Neural Network with Numpy

This is an implementation of a feed forward neural network using numpy. This model uses a dataset containing the pixel values of clothing images and categorize them into 10 different classes.

In [1]:
import numpy as np
import pandas as pd
import matplotlib.pyplot as plt
import math
In [25]:
train_data = pd.read_csv('trainData.csv', header=None)
train_lb = pd.read_csv('trainLabels.csv', header=None)
test_data = pd.read_csv('testData.csv', header=None)
test_lb = pd.read_csv('testLabels.csv', header=None)
train_data
Out[25]:
0 1 2 3 4 5 6 7 8 9 ... 774 775 776 777 778 779 780 781 782 783
0 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
1 0 0 0 0 0 1 0 0 0 0 ... 119 114 130 76 0 0 0 0 0 0
2 0 0 0 0 0 0 0 0 0 22 ... 0 0 1 0 0 0 0 0 0 0
3 0 0 0 0 0 0 0 0 33 96 ... 0 0 0 0 0 0 0 0 0 0
4 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ... ...
59995 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
59996 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0
59997 0 0 0 0 0 0 0 0 0 5 ... 0 0 0 0 0 0 0 0 0 0
59998 0 0 0 0 0 0 0 0 0 0 ... 66 54 50 5 0 1 0 0 0 0
59999 0 0 0 0 0 0 0 0 0 0 ... 0 0 0 0 0 0 0 0 0 0

60000 rows × 784 columns

In [42]:
indexes = []
for i in range(10):
    indexes.append(train_lb.index[train_lb[0] == i].tolist()[0])

for i in range(10):
    x = np.array(train_data.iloc[i])
    x = x.reshape(28,28)
    plt.figure(figsize=(2, 2))
    plt.imshow(x, cmap='gray')
    plt.xlabel(str(i+1)+' th class')
    plt.show()
In [50]:
train_values = []
test_values = []
for i in range(10) :
    train_values.append(train_lb.loc[ train_lb[0] == i ].shape[0])
    test_values.append(test_lb.loc[test_lb[0] == i].shape[0])
plt.figure(figsize=(8, 8))
plt.bar([i for i in range(10)], train_values, label='train', width=0.3)
plt.bar([i+0.3 for i in range(10)], test_values, label='test', width=0.3)
plt.legend()
plt.show()
[6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000, 6000]
[1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000, 1000]

Normalization makes all the data to be between the range of 0 to 1 this way we can benefit from working with smaller numbers which requires less process and also prevent gradian explosion

In [8]:
t = train_data / 255
t.to_csv('NtrainData.csv', headers = None, )

Dataloader

In [2]:
class Dataloader:
    
    def __init__(self, data, labels, n_classes, batch_size=None, shuffle=False):
        
        assert len(data)==len(labels)
        self.__n_classes = n_classes
        self.__batch_size = batch_size
        self.__shuffle = shuffle
        self.__data = data
        self.__onehot_labels = self.__onehot(labels, self.__n_classes)
    
    def __onehot(self, labels, n_classes):
        labels_changed = labels.reshape(-1)
        n_values = np.max(labels_changed) + 1
        onehot_vectors = np.eye(n_values)[labels_changed]
        return onehot_vectors
    
    def __shuffle_dataset(self):
        order = np.random.permutation(len(self.__data))
        self.__data = self.__data[order]
        self.__onehot_labels = self.__onehot_labels[order]
    
    def __iter__(self):
        if self.__shuffle:
            self.__shuffle_dataset()
            
        if self.__batch_size==None:
            yield (np.matrix(self.__data), np.matrix(self.__onehot_labels))
            return
            
        for idx in range(0, len(self.__data), self.__batch_size):
            yield (np.matrix(self.__data[idx:idx+self.__batch_size]), 
                   np.matrix(self.__onehot_labels[idx:idx+self.__batch_size]))

Activation Functions

In [3]:
class Identical:
    
    def __init__(self): pass
    
    def __val(self, matrix):
        identical_value = np.matrix(matrix, dtype=float)
        return identical_value

    def derivative(self, matrix):
        temp = np.matrix(matrix, dtype=float)
        identical_derivative = np.matrix(np.full(np.shape(temp), 1.))
        return identical_derivative
    
    def __call__(self, matrix):
        return self.__val(matrix)
    

class Relu:
    
    def __init__(self): pass
        
    def __f(self, x):
        return max(0, x)

    def __df(self, x):
        return 0 if x < 0 else 1   
    
    def __relu(self, matrix):
        func = np.vectorize(self.__f)
        relu_value = func(matrix)
        return relu_value

    def derivative(self, matrix):
        func = np.vectorize(self.__df)
        relu_derivative = func(matrix)
        return relu_derivative
    
    def __call__(self, matrix):
        return self.__relu(matrix)

    
class LeakyRelu:
    
    def __init__(self, negative_slope=0.01):
        self.negative_slope = 0.01
    
    def __f(self, x):
        return x if x >= 0 else x*self.negative_slope

    def __df(self, x):
        return 1 if x >= 0 else self.negative_slope
    
    def __val(self, matrix):
        func = np.vectorize(self.__f)
        leacky_relu_value = func(matrix)
        return leacky_relu_value

    def derivative(self, matrix):
        func = np.vectorize(self.__df)
        leacky_relu_derivative = func(matrix)
        return leacky_relu_derivative
    
    def __call__(self, matrix):
        return self.__val(matrix)

    
class Sigmoid:
    
    def __init__(self): pass

    def __f(self, x):
        return 1/(1+math.exp(-x))

    def __df(self, x):
        return self.__f(x)*(1-self.__f(x))

    def __val(self, matrix):
        func = np.vectorize(self.__f)
        sigmoid_value = func(matrix)
        return sigmoid_value

    def derivative(self, matrix):
        func = np.vectorize(self.__df)
        sigmoid_derivative = func(matrix)
        return sigmoid_derivative
    
    def __call__(self, matrix):
        return self.__val(matrix)


class Softmax:
    
    def __init__(self): pass

    def __val(self, matrix):
        mx = matrix.max(axis=1)
        numerator = np.exp(matrix - mx)
        denominator = np.sum(numerator, axis=-1)
        softmax_value = numerator/denominator
        return softmax_value

    def derivative(self, matrix):
        # TODO: Implement
        return softmax_derivative
    
    def __call__(self, matrix):
        return self.__val(matrix)

Loss Function

In [4]:
class CrossEntropy: #(with softmax)
    
    def __init__(self): pass

    def __val(self, true_val, expected_val):
        assert np.shape(true_val)==np.shape(expected_val)
        softmax_output = Softmax()(true_val)
        cross_entropy_value = -1*np.multiply(np.log(softmax_output), expected_val).sum(1)
        return cross_entropy_value
        
    def derivative(self, true_val, expected_val):
        assert np.shape(true_val)==np.shape(expected_val)
        cross_entropy_derivative = Softmax()(true_val) - expected_val
        return cross_entropy_derivative
    
    def __call__(self, true_val, expected_val):
        return self.__val(true_val, expected_val)

Layer

In [71]:
class Layer:

    DEFAULT_LOW, DEFAULT_HIGH, DEFAULT_MEAN, DEFAULT_VAR = 0, 0.05, 0., 1.
  
    def __init__(self, input_size, output_size, 
                 activation=Identical(), initial_weight='uniform', **initializing_parameters):
        
        assert type(initial_weight)==str, 'Undefined activation function!'
        self.__weight_initializer_dict = {'uniform':self.__uniform_weight, 'normal':self.__normal_weight}
        assert initial_weight in self.__weight_initializer_dict, 'Undefined weight initialization function!'
        
        self.__n_neurons = output_size
        weight_initializer = self.__weight_initializer_dict[initial_weight]
        self.__weight = weight_initializer(input_size, self.__n_neurons, **initializing_parameters)
        self.__bias = weight_initializer(1, self.__n_neurons, **initializing_parameters)
        self.__activation = activation
        
        self.__last_input = None
        self.__last_activation_input = None
        self.__last_activation_output = None
        self.__last_activation_derivative = None
        
    def forward(self, layer_input):
        assert np.ndim(layer_input)==2
        assert np.size(self.__weight,0) == np.size(layer_input,1)
        self.__last_input = layer_input
        self.__last_activation_input = np.matmul(layer_input, self.__weight)
        self.__last_activation_input = self.__last_activation_input + self.__bias
        self.__last_activation_output = self.__activation(self.__last_activation_input)
        self.__last_activation_derivative = self.__activation.derivative(self.__last_activation_input)
        return self.__last_activation_output
    
    def update_weights(self, backprop_tensor, lr):
        assert np.ndim(backprop_tensor)==2
        assert np.size(backprop_tensor,0) == np.size(self.__last_activation_derivative,0)
        assert np.size(backprop_tensor,1) == self.__n_neurons

        dEdY = np.multiply(backprop_tensor, self.__last_activation_derivative)
        dEdW = np.matmul(np.transpose(self.__last_input), dEdY)
        backprop_tensor = np.matmul(dEdY, np.transpose(self.__weight))
        self.__weight = self.__weight - lr*dEdW

        dEdB = dEdY.sum(axis = 1)
        self.__bias = self.__bias - lr*dEdB
        return backprop_tensor

    def __uniform_weight(self, dim1, dim2, **initializing_parameters):
        low, high = self.DEFAULT_LOW, self.DEFAULT_HIGH
        if 'low' in initializing_parameters.keys(): low = initializing_parameters['low']
        if 'high' in initializing_parameters.keys(): high = initializing_parameters['high']
        weights = np.random.uniform(low=low, high=high, size=(dim1, dim2))
        return weights

    def __normal_weight(self, dim1, dim2, **initializing_parameters):
        mean, var = self.DEFAULT_MEAN, self.DEFAULT_VAR
        if 'mean' in initializing_parameters.keys(): mean = initializing_parameters['mean']
        if 'var' in initializing_parameters.keys(): var = initializing_parameters['var']
        weights = np.random.normal(mean, np.sqrt(var), (dim1, dim2))
        return weights
    
    def get_output(self):
        return self.__last_activation_output

    @property
    def n_neurons(self): return self.__n_neurons
    
    @property
    def weight(self): return self.__weight
    
    @property
    def bias(self): return self.__bias
    
    @property
    def activation(self): return self.__activation

Feed Forward Neural Network

In [72]:
class FeedForwardNN:
    
    def __init__(self, input_shape):
        
        self.__input_shape = input_shape
        self.__output_shape = None
        
        self.__layers_list = []
        
        self.__lr = None
        self.__loss = None

        self.train_output = []
        self.test_output = []

        
    def add_layer(self, n_neurons, activation=Relu(), initial_weight='uniform', **initializing_parameters):
         
        assert type(n_neurons)==int, "Invalid number of neurons for the layer!"
        assert n_neurons>0, "Invalid number of neurons for the layer!"
        
        n_prev_neurons = self.__input_shape if len(self.__layers_list)==0 else self.__layers_list[-1].n_neurons
        new_layer = Layer(n_prev_neurons, n_neurons, activation, initial_weight, **initializing_parameters)
        self.__layers_list.append(new_layer)
        self.__output_shape = self.__layers_list[-1].n_neurons 
      
    
    def set_training_param(self, loss=CrossEntropy(), lr=1e-3):
        assert self.__layers_list, "Uncomplete model!"
        self.__loss = loss
        self.__lr = lr
    
    
    def forward(self, network_input):
        assert type(self.__output_shape) != None, "Model is not compiled!"        
        inp = network_input
        for layer in self.__layers_list:
            inp = layer.forward(inp)
        network_output = inp
        return network_output
    
    
    def fit(self, epochs, trainloader, testloader=None, print_results=True):
        
        assert type(self.__output_shape) != None, "Model is not compiled!"
        assert type(self.__lr) != None and type(self.__loss) != None, "Training paramenters are not set!"

        log = {"train_accuracy":[], "train_loss":[], "test_accuracy":[], "test_loss":[]}
        
        for epoch in range(1, epochs+1):
            
            if print_results: 
                print('Epoch {}:'.format(epoch)) 

            self.train_output = []
            self.test_output = []
                
            average_accuracy, average_loss = self.__train(trainloader)
            log['train_accuracy'].append(average_accuracy)
            log['train_loss'].append(average_loss)
            if print_results:
                print('\tTrain: Average Accuracy: {}\tAverage Loss: {}'.format(average_accuracy, average_loss))
            
            if type(testloader) != type(None):
                average_accuracy, average_loss = self.__test(testloader)
                log['test_accuracy'].append(average_accuracy)
                log['test_loss'].append(average_loss)
                if print_results:
                    print('\tTest: Average Accuracy: {}\tAverage Loss: {}'.format(average_accuracy, average_loss))
                    
        return log
    
    
    def __train(self, trainloader):
        bach_accuracies, batch_losses = [], []
        size = -1
        for x_train, y_train in trainloader:
            if size == -1:
                size = x_train.shape[0]
            if x_train.shape[0] != size:
                break
            batch_accuracy, batch_loss = self.__train_on_batch(x_train, y_train)
            bach_accuracies.append(batch_accuracy)
            batch_losses.append(batch_loss)
        return np.mean(bach_accuracies), np.mean(batch_losses)
    
    
    def __test(self, testloader):
        bach_accuracies, batch_losses = [], []
        size = -1
        for x_test, y_test in testloader:
            if size == -1:
                size = x_test.shape[0]
            if x_test.shape[0] != size:
                break
            batch_accuracy, batch_loss = self.__test_on_batch(x_test, y_test)
            bach_accuracies.append(batch_accuracy)
            batch_losses.append(batch_loss)
        return np.mean(bach_accuracies), np.mean(batch_losses)

    
    def __train_on_batch(self, x_batch, y_batch):
        output = self.forward(x_batch)
        self.__update_weights(output, y_batch)
        batch_accuracy = self.__compute_accuracy(output, y_batch)
        batch_average_loss = np.mean(self.__loss(output, y_batch))

        self.train_output.append((self.__layers_list[-2].get_output(), y_batch))

        return (batch_accuracy, batch_average_loss)
        
        
    def __test_on_batch(self, x_batch, y_batch):
        output = self.forward(x_batch)
        batch_accuracy = self.__compute_accuracy(output, y_batch)
        batch_average_loss = np.mean(self.__loss(output, y_batch))

        self.test_output.append((self.__layers_list[-2].get_output(), y_batch))

        return (batch_accuracy, batch_average_loss)

    def get_outputs(self):
        return self.train_output, self.test_output
            
        
    def __get_labels(self, outputs):
        # TODO: Implement
        return labels
    
    
    def __compute_accuracy(self, output, expected_output):
        true_labels = np.argmax(output, axis=1)
        expected_labels = np.argmax(expected_output, axis=1)
        assert len(true_labels) == len(expected_labels)
        accuracy = (true_labels == expected_labels).sum() / len(true_labels)
        return accuracy
    
    
    def __update_weights(self, output, y_train):
        backprob_tensor = self.__loss.derivative(output, y_train)
        for layer in reversed(self.__layers_list):
            backprob_tensor = layer.update_weights(backprob_tensor, self.__lr)
        return

Training Sample Code

Loading and normalizing data

In [7]:
train_data = np.loadtxt(open("trainData.csv", "rb"), delimiter=",")
train_labels = np.loadtxt(open("trainLabels.csv", "rb"), delimiter=",")
test_data = np.loadtxt(open("testData.csv", "rb"), delimiter=",")
test_labels = np.loadtxt(open("testLabels.csv", "rb"), delimiter=",")
In [8]:
ntrain_data = train_data / 255
ntest_data = test_data / 255
train_labels = train_labels.astype('int64')
test_labels = test_labels.astype('int64')

First tests

In [9]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-3
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation=LeakyRelu(), weight_initializer='uniform')
network.add_layer(10, activation=LeakyRelu(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.1016	Average Loss: 2.303680558715418
	Test: Average Accuracy: 0.1759815705128205	Average Loss: 2.286143790941447
Epoch 2:
	Train: Average Accuracy: 0.12085	Average Loss: 2.281920567023503
	Test: Average Accuracy: 0.10717147435897435	Average Loss: 2.2747059634176545
Epoch 3:
	Train: Average Accuracy: 0.11746666666666666	Average Loss: 2.2673658952396276
	Test: Average Accuracy: 0.14733573717948717	Average Loss: 2.258680482511617
Epoch 4:
	Train: Average Accuracy: 0.11275	Average Loss: 2.2495971297522357
	Test: Average Accuracy: 0.10016025641025642	Average Loss: 2.238124084255277
Epoch 5:
	Train: Average Accuracy: 0.10413333333333333	Average Loss: 2.224261378379141
	Test: Average Accuracy: 0.10697115384615384	Average Loss: 2.207060253111821
Epoch 6:
	Train: Average Accuracy: 0.10248333333333333	Average Loss: 2.191978404986267
	Test: Average Accuracy: 0.12920673076923078	Average Loss: 2.1666944352726194
Epoch 7:
	Train: Average Accuracy: 0.11471666666666666	Average Loss: 2.145053410814865
	Test: Average Accuracy: 0.10186298076923077	Average Loss: 2.116236574303315
Epoch 8:
	Train: Average Accuracy: 0.23988333333333334	Average Loss: 1.8849047793300766
	Test: Average Accuracy: 0.36047676282051283	Average Loss: 1.6991070235638979
Epoch 9:
	Train: Average Accuracy: 0.49255	Average Loss: 1.4781548106648177
	Test: Average Accuracy: 0.6138822115384616	Average Loss: 1.207786692104944
Epoch 10:
	Train: Average Accuracy: 0.6454333333333333	Average Loss: 1.0883633362307865
	Test: Average Accuracy: 0.7055288461538461	Average Loss: 0.9945365148265981
Epoch 11:
	Train: Average Accuracy: 0.7964166666666667	Average Loss: 0.6047698474686426
	Test: Average Accuracy: 0.8131009615384616	Average Loss: 0.5501947340828889
Epoch 12:
	Train: Average Accuracy: 0.8316833333333333	Average Loss: 0.4960001579207658
	Test: Average Accuracy: 0.8341346153846154	Average Loss: 0.4897620218353886
Epoch 13:
	Train: Average Accuracy: 0.8448833333333333	Average Loss: 0.45750843961633825
	Test: Average Accuracy: 0.8303285256410257	Average Loss: 0.4968763949871116
Epoch 14:
	Train: Average Accuracy: 0.84975	Average Loss: 0.43518167388348883
	Test: Average Accuracy: 0.8336338141025641	Average Loss: 0.47698044682103957
Epoch 15:
	Train: Average Accuracy: 0.8541	Average Loss: 0.4200941187868541
	Test: Average Accuracy: 0.832832532051282	Average Loss: 0.47258911156037026
Epoch 16:
	Train: Average Accuracy: 0.8593833333333334	Average Loss: 0.40709610857261497
	Test: Average Accuracy: 0.8393429487179487	Average Loss: 0.4521947041044628
Epoch 17:
	Train: Average Accuracy: 0.8600166666666667	Average Loss: 0.39826559131298056
	Test: Average Accuracy: 0.8481570512820513	Average Loss: 0.43241423021542885
Epoch 18:
	Train: Average Accuracy: 0.8630833333333333	Average Loss: 0.39047823036356477
	Test: Average Accuracy: 0.8494591346153846	Average Loss: 0.42523142711253226
Epoch 19:
	Train: Average Accuracy: 0.8659	Average Loss: 0.38097767076897143
	Test: Average Accuracy: 0.8515625	Average Loss: 0.4276898316540766
Epoch 20:
	Train: Average Accuracy: 0.8676333333333334	Average Loss: 0.37421535087005675
	Test: Average Accuracy: 0.8477564102564102	Average Loss: 0.42893859791018824
Epoch 21:
	Train: Average Accuracy: 0.8696833333333334	Average Loss: 0.3672786151910691
	Test: Average Accuracy: 0.84765625	Average Loss: 0.423955781419841
Epoch 22:
	Train: Average Accuracy: 0.8717166666666667	Average Loss: 0.3627565728386563
	Test: Average Accuracy: 0.8500600961538461	Average Loss: 0.42494071051756777
Epoch 23:
	Train: Average Accuracy: 0.8723333333333333	Average Loss: 0.3578737146353873
	Test: Average Accuracy: 0.8526642628205128	Average Loss: 0.42132922645320353
Epoch 24:
	Train: Average Accuracy: 0.87585	Average Loss: 0.3527448904632056
	Test: Average Accuracy: 0.8458533653846154	Average Loss: 0.42675680593570803
Epoch 25:
	Train: Average Accuracy: 0.8765166666666667	Average Loss: 0.3474959124371357
	Test: Average Accuracy: 0.8551682692307693	Average Loss: 0.41529237086726056
Epoch 26:
	Train: Average Accuracy: 0.8780166666666667	Average Loss: 0.3425227784540144
	Test: Average Accuracy: 0.8580729166666666	Average Loss: 0.40702228391352396
Epoch 27:
	Train: Average Accuracy: 0.8783166666666666	Average Loss: 0.33923740358956983
	Test: Average Accuracy: 0.8571714743589743	Average Loss: 0.4071966933573103
Epoch 28:
	Train: Average Accuracy: 0.8799666666666667	Average Loss: 0.33626858552060684
	Test: Average Accuracy: 0.8574719551282052	Average Loss: 0.40915021635153087
Epoch 29:
	Train: Average Accuracy: 0.8808166666666667	Average Loss: 0.33421473247115374
	Test: Average Accuracy: 0.8623798076923077	Average Loss: 0.3888501322687454
Epoch 30:
	Train: Average Accuracy: 0.88125	Average Loss: 0.33091618886248453
	Test: Average Accuracy: 0.8566706730769231	Average Loss: 0.40899730528855466
In [10]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-2
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation=LeakyRelu(), weight_initializer='uniform')
network.add_layer(10, activation=LeakyRelu(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.296	Average Loss: 1.8647048296442081
	Test: Average Accuracy: 0.5904447115384616	Average Loss: 1.121416934264766
Epoch 2:
	Train: Average Accuracy: 0.6590333333333334	Average Loss: 0.9423386320651389
	Test: Average Accuracy: 0.7305689102564102	Average Loss: 0.7604583227325075
Epoch 3:
	Train: Average Accuracy: 0.7625166666666666	Average Loss: 0.689265072577439
	Test: Average Accuracy: 0.7792467948717948	Average Loss: 0.6576710002445446
Epoch 4:
	Train: Average Accuracy: 0.7972166666666667	Average Loss: 0.5899394244713241
	Test: Average Accuracy: 0.7930689102564102	Average Loss: 0.619184377087444
Epoch 5:
	Train: Average Accuracy: 0.81035	Average Loss: 0.5431638896485333
	Test: Average Accuracy: 0.8092948717948718	Average Loss: 0.5765289375898439
Epoch 6:
	Train: Average Accuracy: 0.82185	Average Loss: 0.5113173861972695
	Test: Average Accuracy: 0.8136017628205128	Average Loss: 0.5738799809092785
Epoch 7:
	Train: Average Accuracy: 0.82805	Average Loss: 0.4929792596992757
	Test: Average Accuracy: 0.7943709935897436	Average Loss: 0.6112207500395991
Epoch 8:
	Train: Average Accuracy: 0.8309	Average Loss: 0.48198557257246344
	Test: Average Accuracy: 0.8030849358974359	Average Loss: 0.574984904359145
Epoch 9:
	Train: Average Accuracy: 0.835	Average Loss: 0.46744837738603456
	Test: Average Accuracy: 0.8229166666666666	Average Loss: 0.5083537775710022
Epoch 10:
	Train: Average Accuracy: 0.8399333333333333	Average Loss: 0.45716199333854857
	Test: Average Accuracy: 0.8218149038461539	Average Loss: 0.5252297303795576
Epoch 11:
	Train: Average Accuracy: 0.84275	Average Loss: 0.44643311878217684
	Test: Average Accuracy: 0.8277243589743589	Average Loss: 0.5044774851198921
Epoch 12:
	Train: Average Accuracy: 0.8429666666666666	Average Loss: 0.44186911032380716
	Test: Average Accuracy: 0.8086939102564102	Average Loss: 0.5457858570525715
Epoch 13:
	Train: Average Accuracy: 0.8470333333333333	Average Loss: 0.43005976963861786
	Test: Average Accuracy: 0.8365384615384616	Average Loss: 0.4769405122182374
Epoch 14:
	Train: Average Accuracy: 0.8493166666666667	Average Loss: 0.4233740085225868
	Test: Average Accuracy: 0.8223157051282052	Average Loss: 0.49509099811276785
Epoch 15:
	Train: Average Accuracy: 0.8524333333333334	Average Loss: 0.4176567949637066
	Test: Average Accuracy: 0.8372395833333334	Average Loss: 0.4709781192106206
Epoch 16:
	Train: Average Accuracy: 0.8539333333333333	Average Loss: 0.411637873182028
	Test: Average Accuracy: 0.8237179487179487	Average Loss: 0.519476289639611
Epoch 17:
	Train: Average Accuracy: 0.85415	Average Loss: 0.4072429225415794
	Test: Average Accuracy: 0.8323317307692307	Average Loss: 0.49521226030189025
Epoch 18:
	Train: Average Accuracy: 0.8558833333333333	Average Loss: 0.40154442859235334
	Test: Average Accuracy: 0.8212139423076923	Average Loss: 0.5374346877551406
Epoch 19:
	Train: Average Accuracy: 0.85675	Average Loss: 0.3974690888447845
	Test: Average Accuracy: 0.8345352564102564	Average Loss: 0.48541742357986634
Epoch 20:
	Train: Average Accuracy: 0.8594166666666667	Average Loss: 0.3940659735732465
	Test: Average Accuracy: 0.8395432692307693	Average Loss: 0.4691334246089681
Epoch 21:
	Train: Average Accuracy: 0.86025	Average Loss: 0.3873595134045
	Test: Average Accuracy: 0.8399439102564102	Average Loss: 0.48185662293714426
Epoch 22:
	Train: Average Accuracy: 0.8628666666666667	Average Loss: 0.3836054984810012
	Test: Average Accuracy: 0.8397435897435898	Average Loss: 0.47029831173145326
Epoch 23:
	Train: Average Accuracy: 0.86285	Average Loss: 0.3793774263124162
	Test: Average Accuracy: 0.8416466346153846	Average Loss: 0.4656533255032797
Epoch 24:
	Train: Average Accuracy: 0.8631166666666666	Average Loss: 0.37743033444779683
	Test: Average Accuracy: 0.8359375	Average Loss: 0.5227936764940825
Epoch 25:
	Train: Average Accuracy: 0.8631333333333333	Average Loss: 0.3765012070814334
	Test: Average Accuracy: 0.8467548076923077	Average Loss: 0.46673882544351597
Epoch 26:
	Train: Average Accuracy: 0.8655666666666667	Average Loss: 0.3708388696238296
	Test: Average Accuracy: 0.8453525641025641	Average Loss: 0.46957655723109837
Epoch 27:
	Train: Average Accuracy: 0.8661833333333333	Average Loss: 0.36780822937367624
	Test: Average Accuracy: 0.8462540064102564	Average Loss: 0.46697126443577364
Epoch 28:
	Train: Average Accuracy: 0.8674833333333334	Average Loss: 0.36705753459625357
	Test: Average Accuracy: 0.8424479166666666	Average Loss: 0.4799719909031979
Epoch 29:
	Train: Average Accuracy: 0.8676166666666667	Average Loss: 0.36467740157825956
	Test: Average Accuracy: 0.8331330128205128	Average Loss: 0.5014217022230694
Epoch 30:
	Train: Average Accuracy: 0.8686333333333334	Average Loss: 0.3624724934633989
	Test: Average Accuracy: 0.8481570512820513	Average Loss: 0.45713072225953355
In [11]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-2
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation=LeakyRelu(), weight_initializer='normal')
network.add_layer(10, activation=LeakyRelu(), weight_initializer='normal')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.41645	Average Loss: 1.4610597755551376
	Test: Average Accuracy: 0.5149238782051282	Average Loss: 1.280469736144469
Epoch 2:
	Train: Average Accuracy: 0.6201333333333333	Average Loss: 0.9575558764507822
	Test: Average Accuracy: 0.6623597756410257	Average Loss: 0.8235302321713297
Epoch 3:
	Train: Average Accuracy: 0.65015	Average Loss: 0.9846977975800948
	Test: Average Accuracy: 0.6939102564102564	Average Loss: 0.8522496660191675
Epoch 4:
	Train: Average Accuracy: 0.7141	Average Loss: 0.7916571445144528
	Test: Average Accuracy: 0.6154847756410257	Average Loss: 0.9767128895670291
Epoch 5:
	Train: Average Accuracy: 0.7369	Average Loss: 0.7284742149734278
	Test: Average Accuracy: 0.7464943910256411	Average Loss: 0.6822359567166351
Epoch 6:
	Train: Average Accuracy: 0.7538333333333334	Average Loss: 0.6843729631804568
	Test: Average Accuracy: 0.7555088141025641	Average Loss: 0.6589344380537473
Epoch 7:
	Train: Average Accuracy: 0.7657666666666667	Average Loss: 0.6537972594018511
	Test: Average Accuracy: 0.7688301282051282	Average Loss: 0.6446906671197491
Epoch 8:
	Train: Average Accuracy: 0.7768666666666667	Average Loss: 0.6262403263997349
	Test: Average Accuracy: 0.7778445512820513	Average Loss: 0.616022731812117
Epoch 9:
	Train: Average Accuracy: 0.7872333333333333	Average Loss: 0.5981442439100534
	Test: Average Accuracy: 0.7810496794871795	Average Loss: 0.6387571001346418
Epoch 10:
	Train: Average Accuracy: 0.7940666666666667	Average Loss: 0.57573108648479
	Test: Average Accuracy: 0.7802483974358975	Average Loss: 0.6375242271018512
Epoch 11:
	Train: Average Accuracy: 0.7996666666666666	Average Loss: 0.5590407730250918
	Test: Average Accuracy: 0.8029847756410257	Average Loss: 0.5571362463182836
Epoch 12:
	Train: Average Accuracy: 0.8053833333333333	Average Loss: 0.5458147928507335
	Test: Average Accuracy: 0.8004807692307693	Average Loss: 0.5781536542775103
Epoch 13:
	Train: Average Accuracy: 0.80895	Average Loss: 0.5395609824773354
	Test: Average Accuracy: 0.8042868589743589	Average Loss: 0.5705544498380317
Epoch 14:
	Train: Average Accuracy: 0.8121333333333334	Average Loss: 0.5319245491458068
	Test: Average Accuracy: 0.7962740384615384	Average Loss: 0.5852161663776235
Epoch 15:
	Train: Average Accuracy: 0.8173	Average Loss: 0.5184170455119511
	Test: Average Accuracy: 0.7941706730769231	Average Loss: 0.5939125325250367
Epoch 16:
	Train: Average Accuracy: 0.8207166666666666	Average Loss: 0.5131135703594689
	Test: Average Accuracy: 0.7994791666666666	Average Loss: 0.5764696300156985
Epoch 17:
	Train: Average Accuracy: 0.8238333333333333	Average Loss: 0.502455020944452
	Test: Average Accuracy: 0.815604967948718	Average Loss: 0.5399472548902075
Epoch 18:
	Train: Average Accuracy: 0.8271	Average Loss: 0.4953786195665295
	Test: Average Accuracy: 0.77734375	Average Loss: 0.6757924940105994
Epoch 19:
	Train: Average Accuracy: 0.8269666666666666	Average Loss: 0.48992789554781435
	Test: Average Accuracy: 0.8158052884615384	Average Loss: 0.5483688824810616
Epoch 20:
	Train: Average Accuracy: 0.8305833333333333	Average Loss: 0.4848332917533741
	Test: Average Accuracy: 0.8230168269230769	Average Loss: 0.5289011748284387
Epoch 21:
	Train: Average Accuracy: 0.8335	Average Loss: 0.47502035983862595
	Test: Average Accuracy: 0.8173076923076923	Average Loss: 0.5470730093410514
Epoch 22:
	Train: Average Accuracy: 0.8351166666666666	Average Loss: 0.4704916731084062
	Test: Average Accuracy: 0.8214142628205128	Average Loss: 0.526245542616558
Epoch 23:
	Train: Average Accuracy: 0.8383333333333334	Average Loss: 0.4677080383289651
	Test: Average Accuracy: 0.8288261217948718	Average Loss: 0.5117697781607541
Epoch 24:
	Train: Average Accuracy: 0.83885	Average Loss: 0.4623001871264189
	Test: Average Accuracy: 0.8113982371794872	Average Loss: 0.5536868783557518
Epoch 25:
	Train: Average Accuracy: 0.8394	Average Loss: 0.46239888902719417
	Test: Average Accuracy: 0.7871594551282052	Average Loss: 0.6485160544828775
Epoch 26:
	Train: Average Accuracy: 0.8432166666666666	Average Loss: 0.4503794793045417
	Test: Average Accuracy: 0.8275240384615384	Average Loss: 0.5165807996735936
Epoch 27:
	Train: Average Accuracy: 0.8427333333333333	Average Loss: 0.4510031558819993
	Test: Average Accuracy: 0.8304286858974359	Average Loss: 0.5082733903718641
Epoch 28:
	Train: Average Accuracy: 0.8442666666666667	Average Loss: 0.445958929332459
	Test: Average Accuracy: 0.8314302884615384	Average Loss: 0.5059666821108604
Epoch 29:
	Train: Average Accuracy: 0.8456666666666667	Average Loss: 0.44122431555665625
	Test: Average Accuracy: 0.8326322115384616	Average Loss: 0.5080633402951003
Epoch 30:
	Train: Average Accuracy: 0.8479666666666666	Average Loss: 0.4357452807555282
	Test: Average Accuracy: 0.8300280448717948	Average Loss: 0.5139375307410143

testing with relu as the activation fucntion and different learning rates

In [12]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-2
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation= Relu(), weight_initializer='uniform')
network.add_layer(10, activation=Relu(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.24038333333333334	Average Loss: 2.030406851244226
	Test: Average Accuracy: 0.35396634615384615	Average Loss: 1.7677377121148798
Epoch 2:
	Train: Average Accuracy: 0.3609833333333333	Average Loss: 1.7286198845923555
	Test: Average Accuracy: 0.36538461538461536	Average Loss: 1.7221738496722447
Epoch 3:
	Train: Average Accuracy: 0.37205	Average Loss: 1.6948512374799212
	Test: Average Accuracy: 0.3717948717948718	Average Loss: 1.7008812361970786
Epoch 4:
	Train: Average Accuracy: 0.4031	Average Loss: 1.5784623013449706
	Test: Average Accuracy: 0.39903846153846156	Average Loss: 1.420611593840557
Epoch 5:
	Train: Average Accuracy: 0.38148333333333334	Average Loss: 1.498960319089471
	Test: Average Accuracy: 0.3762019230769231	Average Loss: 1.4415585204603383
Epoch 6:
	Train: Average Accuracy: 0.44656666666666667	Average Loss: 1.3799253570905765
	Test: Average Accuracy: 0.5418669871794872	Average Loss: 1.2433399141888413
Epoch 7:
	Train: Average Accuracy: 0.52915	Average Loss: 1.268313745916077
	Test: Average Accuracy: 0.45783253205128205	Average Loss: 1.278625215545064
Epoch 8:
	Train: Average Accuracy: 0.5784	Average Loss: 1.099734972609164
	Test: Average Accuracy: 0.5994591346153846	Average Loss: 0.9770380778667705
Epoch 9:
	Train: Average Accuracy: 0.63315	Average Loss: 0.9554874524470955
	Test: Average Accuracy: 0.6308092948717948	Average Loss: 0.9754214775629965
Epoch 10:
	Train: Average Accuracy: 0.64075	Average Loss: 0.9336500863813276
	Test: Average Accuracy: 0.6739783653846154	Average Loss: 0.8682630521402049
Epoch 11:
	Train: Average Accuracy: 0.6464166666666666	Average Loss: 0.9210258071034727
	Test: Average Accuracy: 0.5990584935897436	Average Loss: 1.0782414707667771
Epoch 12:
	Train: Average Accuracy: 0.6618166666666667	Average Loss: 0.8864143034404924
	Test: Average Accuracy: 0.6972155448717948	Average Loss: 0.8170013908132402
Epoch 13:
	Train: Average Accuracy: 0.6907833333333333	Average Loss: 0.8369839133335941
	Test: Average Accuracy: 0.6755809294871795	Average Loss: 0.823471160602246
Epoch 14:
	Train: Average Accuracy: 0.6994666666666667	Average Loss: 0.8261509808485896
	Test: Average Accuracy: 0.7058293269230769	Average Loss: 0.8087962691585306
Epoch 15:
	Train: Average Accuracy: 0.7045	Average Loss: 0.8062461321187618
	Test: Average Accuracy: 0.7236578525641025	Average Loss: 0.7604146208991479
Epoch 16:
	Train: Average Accuracy: 0.7131333333333333	Average Loss: 0.7837408040627183
	Test: Average Accuracy: 0.729667467948718	Average Loss: 0.770575456011812
Epoch 17:
	Train: Average Accuracy: 0.7136666666666667	Average Loss: 0.7798221021205792
	Test: Average Accuracy: 0.710136217948718	Average Loss: 0.7774666862815134
Epoch 18:
	Train: Average Accuracy: 0.7175833333333334	Average Loss: 0.7663309120557708
	Test: Average Accuracy: 0.6798878205128205	Average Loss: 0.8713957690737485
Epoch 19:
	Train: Average Accuracy: 0.7179333333333333	Average Loss: 0.7644094817958403
	Test: Average Accuracy: 0.7332732371794872	Average Loss: 0.7383646164575643
Epoch 20:
	Train: Average Accuracy: 0.75125	Average Loss: 0.7368559153595284
	Test: Average Accuracy: 0.7564102564102564	Average Loss: 0.6833779373044813
Epoch 21:
	Train: Average Accuracy: 0.80205	Average Loss: 0.6129254638133943
	Test: Average Accuracy: 0.8253205128205128	Average Loss: 0.5611005413442501
Epoch 22:
	Train: Average Accuracy: 0.8077166666666666	Average Loss: 0.6009870814524103
	Test: Average Accuracy: 0.7828525641025641	Average Loss: 0.6107100282732757
Epoch 23:
	Train: Average Accuracy: 0.8091833333333334	Average Loss: 0.5879540601415424
	Test: Average Accuracy: 0.8262219551282052	Average Loss: 0.529820241568976
Epoch 24:
	Train: Average Accuracy: 0.8184333333333333	Average Loss: 0.5490237177258336
	Test: Average Accuracy: 0.8353365384615384	Average Loss: 0.5338268197451992
Epoch 25:
	Train: Average Accuracy: 0.8289	Average Loss: 0.5033700895824694
	Test: Average Accuracy: 0.7998798076923077	Average Loss: 0.6030106504552872
Epoch 26:
	Train: Average Accuracy: 0.8348666666666666	Average Loss: 0.4823098470195198
	Test: Average Accuracy: 0.8261217948717948	Average Loss: 0.5348302086873568
Epoch 27:
	Train: Average Accuracy: 0.83905	Average Loss: 0.4681556375853332
	Test: Average Accuracy: 0.827323717948718	Average Loss: 0.4989376080445618
Epoch 28:
	Train: Average Accuracy: 0.8429	Average Loss: 0.45673371842497595
	Test: Average Accuracy: 0.8165064102564102	Average Loss: 0.5237872275467128
Epoch 29:
	Train: Average Accuracy: 0.8456333333333333	Average Loss: 0.45100975669144583
	Test: Average Accuracy: 0.8382411858974359	Average Loss: 0.5116521439883375
Epoch 30:
	Train: Average Accuracy: 0.8474833333333334	Average Loss: 0.446664453664884
	Test: Average Accuracy: 0.8232171474358975	Average Loss: 0.5372172995420543
In [13]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-1
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation= Relu(), weight_initializer='uniform')
network.add_layer(10, activation=Relu(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.10105	Average Loss: nan
	Test: Average Accuracy: 0.10216346153846154	Average Loss: 2.302473338472399
Epoch 2:
	Train: Average Accuracy: 0.10021666666666666	Average Loss: 2.302607723806549
	Test: Average Accuracy: 0.10306490384615384	Average Loss: 2.3026500008322635
Epoch 3:
	Train: Average Accuracy: 0.10053333333333334	Average Loss: 2.302650164021695
	Test: Average Accuracy: 0.10106169871794872	Average Loss: 2.3026515336241866
Epoch 4:
	Train: Average Accuracy: 0.10051666666666667	Average Loss: 2.302634041444036
	Test: Average Accuracy: 0.09525240384615384	Average Loss: 2.3025910329635844
Epoch 5:
	Train: Average Accuracy: 0.09903333333333333	Average Loss: 2.3026477010495148
	Test: Average Accuracy: 0.09735576923076923	Average Loss: 2.302692016886203
Epoch 6:
	Train: Average Accuracy: 0.10008333333333333	Average Loss: 2.3026133415447547
	Test: Average Accuracy: 0.09725560897435898	Average Loss: 2.3027150066009066
Epoch 7:
	Train: Average Accuracy: 0.09965	Average Loss: 2.302633502343793
	Test: Average Accuracy: 0.10216346153846154	Average Loss: 2.3026737710569116
Epoch 8:
	Train: Average Accuracy: 0.09958333333333333	Average Loss: 2.302594481754705
	Test: Average Accuracy: 0.09845753205128205	Average Loss: 2.302700081025172
Epoch 9:
	Train: Average Accuracy: 0.09991666666666667	Average Loss: 2.302654583559029
	Test: Average Accuracy: 0.10166266025641026	Average Loss: 2.302604239015673
Epoch 10:
	Train: Average Accuracy: 0.10035	Average Loss: 2.302657069629093
	Test: Average Accuracy: 0.0999599358974359	Average Loss: 2.302689469075693
Epoch 11:
	Train: Average Accuracy: 0.10021666666666666	Average Loss: 2.3026678166610886
	Test: Average Accuracy: 0.09795673076923077	Average Loss: 2.30249393283087
Epoch 12:
	Train: Average Accuracy: 0.10101666666666667	Average Loss: 2.3026218939414425
	Test: Average Accuracy: 0.10366586538461539	Average Loss: 2.3025244781190155
Epoch 13:
	Train: Average Accuracy: 0.09891666666666667	Average Loss: 2.3026489524148404
	Test: Average Accuracy: 0.09785657051282051	Average Loss: 2.3027453268980613
Epoch 14:
	Train: Average Accuracy: 0.09886666666666667	Average Loss: 2.302632183698068
	Test: Average Accuracy: 0.09835737179487179	Average Loss: 2.302540975219493
Epoch 15:
	Train: Average Accuracy: 0.10181666666666667	Average Loss: 2.3026253406453074
	Test: Average Accuracy: 0.09845753205128205	Average Loss: 2.302512690466691
Epoch 16:
	Train: Average Accuracy: 0.10131666666666667	Average Loss: 2.3026068498861694
	Test: Average Accuracy: 0.1006610576923077	Average Loss: 2.3025538766301406
Epoch 17:
	Train: Average Accuracy: 0.10103333333333334	Average Loss: 2.3026213608985926
	Test: Average Accuracy: 0.10126201923076923	Average Loss: 2.302602477192448
Epoch 18:
	Train: Average Accuracy: 0.09823333333333334	Average Loss: 2.3026414299232068
	Test: Average Accuracy: 0.10006009615384616	Average Loss: 2.302611892798459
Epoch 19:
	Train: Average Accuracy: 0.09995	Average Loss: 2.3026263448830164
	Test: Average Accuracy: 0.09755608974358974	Average Loss: 2.302620576688182
Epoch 20:
	Train: Average Accuracy: 0.09995	Average Loss: 2.3026020671427077
	Test: Average Accuracy: 0.10266426282051282	Average Loss: 2.3027460094225907
Epoch 21:
	Train: Average Accuracy: 0.10101666666666667	Average Loss: 2.3026673335914625
	Test: Average Accuracy: 0.10006009615384616	Average Loss: 2.3026031631808745
Epoch 22:
	Train: Average Accuracy: 0.09943333333333333	Average Loss: 2.3026031214485867
	Test: Average Accuracy: 0.09885817307692307	Average Loss: 2.3025460548404797
Epoch 23:
	Train: Average Accuracy: 0.1007	Average Loss: 2.3026455179210443
	Test: Average Accuracy: 0.09875801282051282	Average Loss: 2.3026744726102217
Epoch 24:
	Train: Average Accuracy: 0.1017	Average Loss: 2.3026516711128764
	Test: Average Accuracy: 0.09975961538461539	Average Loss: 2.302640914457477
Epoch 25:
	Train: Average Accuracy: 0.1	Average Loss: 2.302639625520813
	Test: Average Accuracy: 0.10236378205128205	Average Loss: 2.302573921770122
Epoch 26:
	Train: Average Accuracy: 0.1009	Average Loss: 2.3025967552368445
	Test: Average Accuracy: 0.10216346153846154	Average Loss: 2.302718282941571
Epoch 27:
	Train: Average Accuracy: 0.10028333333333334	Average Loss: 2.3026089618772843
	Test: Average Accuracy: 0.10216346153846154	Average Loss: 2.302584694974033
Epoch 28:
	Train: Average Accuracy: 0.10018333333333333	Average Loss: 2.3026635216416333
	Test: Average Accuracy: 0.1038661858974359	Average Loss: 2.3025455782860993
Epoch 29:
	Train: Average Accuracy: 0.1	Average Loss: 2.3026406178046193
	Test: Average Accuracy: 0.09805689102564102	Average Loss: 2.3027177183983887
Epoch 30:
	Train: Average Accuracy: 0.10061666666666666	Average Loss: 2.3026142202942883
	Test: Average Accuracy: 0.09905849358974358	Average Loss: 2.3026855084206552
In [105]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-3
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation= Relu(), weight_initializer='uniform')
network.add_layer(10, activation=Identical(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.6855666666666667	Average Loss: 0.8653604236045115
	Test: Average Accuracy: 0.7938701923076923	Average Loss: 0.5878219256732224
Epoch 2:
	Train: Average Accuracy: 0.8162	Average Loss: 0.5263961981026415
	Test: Average Accuracy: 0.8058894230769231	Average Loss: 0.5502089918679317
Epoch 3:
	Train: Average Accuracy: 0.8332333333333334	Average Loss: 0.4812949221526318
	Test: Average Accuracy: 0.8261217948717948	Average Loss: 0.5065764115737618
Epoch 4:
	Train: Average Accuracy: 0.8411666666666666	Average Loss: 0.458214624097299
	Test: Average Accuracy: 0.8287259615384616	Average Loss: 0.48700957853869387
Epoch 5:
	Train: Average Accuracy: 0.8470333333333333	Average Loss: 0.43798646208696934
	Test: Average Accuracy: 0.8377403846153846	Average Loss: 0.47204780219950293
Epoch 6:
	Train: Average Accuracy: 0.8522166666666666	Average Loss: 0.4247518474430651
	Test: Average Accuracy: 0.8404447115384616	Average Loss: 0.45964455729634096
Epoch 7:
	Train: Average Accuracy: 0.8555	Average Loss: 0.41281345954640175
	Test: Average Accuracy: 0.8434495192307693	Average Loss: 0.46075189075909423
Epoch 8:
	Train: Average Accuracy: 0.85855	Average Loss: 0.4028594766291283
	Test: Average Accuracy: 0.8404447115384616	Average Loss: 0.4553986331716872
Epoch 9:
	Train: Average Accuracy: 0.86095	Average Loss: 0.39379986477149326
	Test: Average Accuracy: 0.8456530448717948	Average Loss: 0.44460848445779133
Epoch 10:
	Train: Average Accuracy: 0.8638333333333333	Average Loss: 0.38852651318103254
	Test: Average Accuracy: 0.8393429487179487	Average Loss: 0.45960982823029256
Epoch 11:
	Train: Average Accuracy: 0.8663	Average Loss: 0.38151383751336554
	Test: Average Accuracy: 0.8531650641025641	Average Loss: 0.4163327132319864
Epoch 12:
	Train: Average Accuracy: 0.8677666666666667	Average Loss: 0.37476082525887694
	Test: Average Accuracy: 0.8455528846153846	Average Loss: 0.44110136727354277
Epoch 13:
	Train: Average Accuracy: 0.8697666666666667	Average Loss: 0.3695011199125321
	Test: Average Accuracy: 0.8515625	Average Loss: 0.4177906437752209
Epoch 14:
	Train: Average Accuracy: 0.8706666666666667	Average Loss: 0.36572700991381063
	Test: Average Accuracy: 0.8479567307692307	Average Loss: 0.4423536105126395
Epoch 15:
	Train: Average Accuracy: 0.8714	Average Loss: 0.3617390333460218
	Test: Average Accuracy: 0.852363782051282	Average Loss: 0.4168019244002433
Epoch 16:
	Train: Average Accuracy: 0.8731666666666666	Average Loss: 0.3569294371025128
	Test: Average Accuracy: 0.8537660256410257	Average Loss: 0.4183425749582423
Epoch 17:
	Train: Average Accuracy: 0.8747	Average Loss: 0.3543036600866657
	Test: Average Accuracy: 0.8547676282051282	Average Loss: 0.4121634761464109
Epoch 18:
	Train: Average Accuracy: 0.8739333333333333	Average Loss: 0.3517538075535271
	Test: Average Accuracy: 0.8539663461538461	Average Loss: 0.41660529251377226
Epoch 19:
	Train: Average Accuracy: 0.8755833333333334	Average Loss: 0.34945238974358556
	Test: Average Accuracy: 0.8579727564102564	Average Loss: 0.40857839862326956
Epoch 20:
	Train: Average Accuracy: 0.8779	Average Loss: 0.3439355350695283
	Test: Average Accuracy: 0.8603766025641025	Average Loss: 0.4005931136438585
Epoch 21:
	Train: Average Accuracy: 0.8773833333333333	Average Loss: 0.34297484789667976
	Test: Average Accuracy: 0.8614783653846154	Average Loss: 0.3971234355254962
Epoch 22:
	Train: Average Accuracy: 0.8786333333333334	Average Loss: 0.3390484656939543
	Test: Average Accuracy: 0.8605769230769231	Average Loss: 0.4044199813666101
Epoch 23:
	Train: Average Accuracy: 0.8804666666666666	Average Loss: 0.33648735811237046
	Test: Average Accuracy: 0.8544671474358975	Average Loss: 0.4153780100799256
Epoch 24:
	Train: Average Accuracy: 0.8804333333333333	Average Loss: 0.3349293671384753
	Test: Average Accuracy: 0.8517628205128205	Average Loss: 0.42531073160299354
Epoch 25:
	Train: Average Accuracy: 0.8803	Average Loss: 0.3331506228671569
	Test: Average Accuracy: 0.8611778846153846	Average Loss: 0.40098127431889763
Epoch 26:
	Train: Average Accuracy: 0.8818666666666667	Average Loss: 0.3294891331629376
	Test: Average Accuracy: 0.8471554487179487	Average Loss: 0.4408242593028098
Epoch 27:
	Train: Average Accuracy: 0.88205	Average Loss: 0.3289364305111314
	Test: Average Accuracy: 0.8592748397435898	Average Loss: 0.3993149096102118
Epoch 28:
	Train: Average Accuracy: 0.8829833333333333	Average Loss: 0.3267944022350862
	Test: Average Accuracy: 0.8449519230769231	Average Loss: 0.44284004044614644
Epoch 29:
	Train: Average Accuracy: 0.8837	Average Loss: 0.32507092584248864
	Test: Average Accuracy: 0.86328125	Average Loss: 0.3910177752512598
Epoch 30:
	Train: Average Accuracy: 0.883	Average Loss: 0.3245074053546148
	Test: Average Accuracy: 0.8589743589743589	Average Loss: 0.3988947887155931

1e-3 and 1e-2 have proven to a proprate learning rate for this model but using 1e-1 is too much for learning rate and model didn't perfome well

using sigmoid and tanh as the activation fucntion

In [106]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-3
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation= Sigmoid(), weight_initializer='uniform')
network.add_layer(10, activation=Identical(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.10568333333333334	Average Loss: 2.303359604301715
	Test: Average Accuracy: 0.16135817307692307	Average Loss: 2.2664916167614373
Epoch 2:
	Train: Average Accuracy: 0.3359333333333333	Average Loss: 1.8532660266464864
	Test: Average Accuracy: 0.46885016025641024	Average Loss: 1.4384914874773467
Epoch 3:
	Train: Average Accuracy: 0.5610666666666667	Average Loss: 1.2192430891664066
	Test: Average Accuracy: 0.6097756410256411	Average Loss: 1.0702055620694082
Epoch 4:
	Train: Average Accuracy: 0.6744833333333333	Average Loss: 0.9485584976177265
	Test: Average Accuracy: 0.7125400641025641	Average Loss: 0.856132929755225
Epoch 5:
	Train: Average Accuracy: 0.728	Average Loss: 0.7720587554212902
	Test: Average Accuracy: 0.7301682692307693	Average Loss: 0.7325681071864965
Epoch 6:
	Train: Average Accuracy: 0.7499833333333333	Average Loss: 0.6792834672852708
	Test: Average Accuracy: 0.7506009615384616	Average Loss: 0.667960792854443
Epoch 7:
	Train: Average Accuracy: 0.7825	Average Loss: 0.6154801832917396
	Test: Average Accuracy: 0.7922676282051282	Average Loss: 0.6148133255590943
Epoch 8:
	Train: Average Accuracy: 0.81225	Average Loss: 0.5654925168452918
	Test: Average Accuracy: 0.8076923076923077	Average Loss: 0.5717334747280989
Epoch 9:
	Train: Average Accuracy: 0.8246333333333333	Average Loss: 0.5314153353201527
	Test: Average Accuracy: 0.8188100961538461	Average Loss: 0.5480278123539467
Epoch 10:
	Train: Average Accuracy: 0.8305833333333333	Average Loss: 0.5077705522577206
	Test: Average Accuracy: 0.8256209935897436	Average Loss: 0.5248468199451364
Epoch 11:
	Train: Average Accuracy: 0.8361833333333333	Average Loss: 0.48917416214400045
	Test: Average Accuracy: 0.8255208333333334	Average Loss: 0.5123185358086286
Epoch 12:
	Train: Average Accuracy: 0.83995	Average Loss: 0.47444107171011585
	Test: Average Accuracy: 0.8293269230769231	Average Loss: 0.5007078901557102
Epoch 13:
	Train: Average Accuracy: 0.8429833333333333	Average Loss: 0.46265252791316597
	Test: Average Accuracy: 0.8240184294871795	Average Loss: 0.49903085385973867
Epoch 14:
	Train: Average Accuracy: 0.84615	Average Loss: 0.45237835441951973
	Test: Average Accuracy: 0.8357371794871795	Average Loss: 0.4844527563932253
Epoch 15:
	Train: Average Accuracy: 0.84855	Average Loss: 0.443856676750818
	Test: Average Accuracy: 0.8391426282051282	Average Loss: 0.4725516769504399
Epoch 16:
	Train: Average Accuracy: 0.8507166666666667	Average Loss: 0.43680256967536607
	Test: Average Accuracy: 0.8397435897435898	Average Loss: 0.4724863302128311
Epoch 17:
	Train: Average Accuracy: 0.852	Average Loss: 0.43044028575452126
	Test: Average Accuracy: 0.840645032051282	Average Loss: 0.4696319384630674
Epoch 18:
	Train: Average Accuracy: 0.8537833333333333	Average Loss: 0.4249319591915452
	Test: Average Accuracy: 0.8410456730769231	Average Loss: 0.46305110768521457
Epoch 19:
	Train: Average Accuracy: 0.8553	Average Loss: 0.4196654444776182
	Test: Average Accuracy: 0.8431490384615384	Average Loss: 0.4591623124188548
Epoch 20:
	Train: Average Accuracy: 0.8569833333333333	Average Loss: 0.41566910091975023
	Test: Average Accuracy: 0.84375	Average Loss: 0.4541984480298277
Epoch 21:
	Train: Average Accuracy: 0.8587166666666667	Average Loss: 0.41178341939010765
	Test: Average Accuracy: 0.8393429487179487	Average Loss: 0.46530475249494874
Epoch 22:
	Train: Average Accuracy: 0.8589	Average Loss: 0.40792754821297156
	Test: Average Accuracy: 0.8454527243589743	Average Loss: 0.45164276286186084
Epoch 23:
	Train: Average Accuracy: 0.8600333333333333	Average Loss: 0.40479684547364253
	Test: Average Accuracy: 0.8474559294871795	Average Loss: 0.4438313949866986
Epoch 24:
	Train: Average Accuracy: 0.8613666666666666	Average Loss: 0.40192739325453525
	Test: Average Accuracy: 0.8486578525641025	Average Loss: 0.44484799092528415
Epoch 25:
	Train: Average Accuracy: 0.86175	Average Loss: 0.3991943649116109
	Test: Average Accuracy: 0.8465544871794872	Average Loss: 0.4439476343051568
Epoch 26:
	Train: Average Accuracy: 0.86335	Average Loss: 0.3969512186550457
	Test: Average Accuracy: 0.8469551282051282	Average Loss: 0.4421858668457553
Epoch 27:
	Train: Average Accuracy: 0.86475	Average Loss: 0.39386719784269797
	Test: Average Accuracy: 0.8486578525641025	Average Loss: 0.43848164532534284
Epoch 28:
	Train: Average Accuracy: 0.8647166666666667	Average Loss: 0.3918844892804535
	Test: Average Accuracy: 0.8498597756410257	Average Loss: 0.4385542827821209
Epoch 29:
	Train: Average Accuracy: 0.86585	Average Loss: 0.38927219017167575
	Test: Average Accuracy: 0.8487580128205128	Average Loss: 0.4360840127714259
Epoch 30:
	Train: Average Accuracy: 0.8677	Average Loss: 0.386986581545965
	Test: Average Accuracy: 0.8463541666666666	Average Loss: 0.43826505211606065
In [20]:
class Tanh:
    
    def __init__(self): pass

    def __f(self, x):
        return 1/(1+math.exp(-x))
    
    def __Tf(self, x):
        return 2/(1+math.exp(-2*x))

    def __df(self, x):
        return 4*self.__f(2*x)*(1-self.__f(2*x))

    def __val(self, matrix):
        func = np.vectorize(self.__Tf)
        sigmoid_value = func(matrix)
        return sigmoid_value

    def derivative(self, matrix):
        func = np.vectorize(self.__df)
        sigmoid_derivative = func(matrix)
        return sigmoid_derivative
    
    def __call__(self, matrix):
        return self.__val(matrix)


INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-2
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation= Tanh(), weight_initializer='uniform')
network.add_layer(10, activation=Tanh(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.10005	Average Loss: 2.3027073743128343
	Test: Average Accuracy: 0.10016025641025642	Average Loss: 2.3024699597442333
Epoch 2:
	Train: Average Accuracy: 0.1	Average Loss: 2.302488257981192
	Test: Average Accuracy: 0.10006009615384616	Average Loss: 2.3023766008258995
Epoch 3:
	Train: Average Accuracy: 0.10166666666666667	Average Loss: 2.2128100044251013
	Test: Average Accuracy: 0.10006009615384616	Average Loss: 1.814206743828389
Epoch 4:
	Train: Average Accuracy: 0.10786666666666667	Average Loss: 1.5530778156439367
	Test: Average Accuracy: 0.16756810897435898	Average Loss: 1.4221891222044196
Epoch 5:
	Train: Average Accuracy: 0.44465	Average Loss: 1.2628679959970186
	Test: Average Accuracy: 0.6572516025641025	Average Loss: 1.1381175464090842
Epoch 6:
	Train: Average Accuracy: 0.6724	Average Loss: 1.1170395432327267
	Test: Average Accuracy: 0.6724759615384616	Average Loss: 1.1195578455022863
Epoch 7:
	Train: Average Accuracy: 0.7106333333333333	Average Loss: 1.093843057444871
	Test: Average Accuracy: 0.7108373397435898	Average Loss: 1.1011231464602282
Epoch 8:
	Train: Average Accuracy: 0.7851333333333333	Average Loss: 1.0612029507516356
	Test: Average Accuracy: 0.7837540064102564	Average Loss: 1.0767252164711885
Epoch 9:
	Train: Average Accuracy: 0.8048333333333333	Average Loss: 1.049203253572956
	Test: Average Accuracy: 0.8041866987179487	Average Loss: 1.0728495616299598
Epoch 10:
	Train: Average Accuracy: 0.82475	Average Loss: 1.0395380185065828
	Test: Average Accuracy: 0.8128004807692307	Average Loss: 1.0677273315084415
Epoch 11:
	Train: Average Accuracy: 0.8293	Average Loss: 1.0335961435673051
	Test: Average Accuracy: 0.8182091346153846	Average Loss: 1.058459711699207
Epoch 12:
	Train: Average Accuracy: 0.83085	Average Loss: 1.0269450905361068
	Test: Average Accuracy: 0.8235176282051282	Average Loss: 1.053568243651777
Epoch 13:
	Train: Average Accuracy: 0.83205	Average Loss: 1.024414062091118
	Test: Average Accuracy: 0.8105969551282052	Average Loss: 1.0533133466958968
Epoch 14:
	Train: Average Accuracy: 0.83715	Average Loss: 1.0206668022131609
	Test: Average Accuracy: 0.7920673076923077	Average Loss: 1.0770523623007047
Epoch 15:
	Train: Average Accuracy: 0.83495	Average Loss: 1.0206222239571012
	Test: Average Accuracy: 0.8330328525641025	Average Loss: 1.03935476932813
Epoch 16:
	Train: Average Accuracy: 0.8404	Average Loss: 1.0155382052130455
	Test: Average Accuracy: 0.8212139423076923	Average Loss: 1.0734062267178839
Epoch 17:
	Train: Average Accuracy: 0.8395666666666667	Average Loss: 1.0133843256488697
	Test: Average Accuracy: 0.8160056089743589	Average Loss: 1.0483394415013554
Epoch 18:
	Train: Average Accuracy: 0.8414166666666667	Average Loss: 1.0111469041014012
	Test: Average Accuracy: 0.83203125	Average Loss: 1.0340198043270266
Epoch 19:
	Train: Average Accuracy: 0.8457	Average Loss: 1.0091659271874591
	Test: Average Accuracy: 0.8168068910256411	Average Loss: 1.0548567555637258
Epoch 20:
	Train: Average Accuracy: 0.8462333333333333	Average Loss: 1.0079891384662354
	Test: Average Accuracy: 0.7725360576923077	Average Loss: 1.0699784239496342
Epoch 21:
	Train: Average Accuracy: 0.84595	Average Loss: 1.0074108863212434
	Test: Average Accuracy: 0.8280248397435898	Average Loss: 1.0363266190919225
Epoch 22:
	Train: Average Accuracy: 0.8474666666666667	Average Loss: 1.0067346378011692
	Test: Average Accuracy: 0.8387419871794872	Average Loss: 1.0323081200241504
Epoch 23:
	Train: Average Accuracy: 0.84615	Average Loss: 1.006273246931843
	Test: Average Accuracy: 0.8379407051282052	Average Loss: 1.0313851345012641
Epoch 24:
	Train: Average Accuracy: 0.8469166666666667	Average Loss: 1.0019586708546036
	Test: Average Accuracy: 0.8310296474358975	Average Loss: 1.048033226712878
Epoch 25:
	Train: Average Accuracy: 0.8490166666666666	Average Loss: 1.0018497822116879
	Test: Average Accuracy: 0.8335336538461539	Average Loss: 1.0335423755807296
Epoch 26:
	Train: Average Accuracy: 0.8465333333333334	Average Loss: 1.0041862250719167
	Test: Average Accuracy: 0.8136017628205128	Average Loss: 1.0504300230542252
Epoch 27:
	Train: Average Accuracy: 0.84935	Average Loss: 0.9999278438019272
	Test: Average Accuracy: 0.8252203525641025	Average Loss: 1.0581496431716055
Epoch 28:
	Train: Average Accuracy: 0.8496666666666667	Average Loss: 0.9991589400604576
	Test: Average Accuracy: 0.8186097756410257	Average Loss: 1.0428587333250452
Epoch 29:
	Train: Average Accuracy: 0.8495833333333334	Average Loss: 0.9979133486079075
	Test: Average Accuracy: 0.8386418269230769	Average Loss: 1.0366164814289396
Epoch 30:
	Train: Average Accuracy: 0.8517666666666667	Average Loss: 0.9964327168728281
	Test: Average Accuracy: 0.8305288461538461	Average Loss: 1.0366264494522268

Sigmoid and Tanh has very small derivaing for too large of too small values and this derivative which will be used for updating the weights become very small and almost makes the weights stop changing so using this activation functions especially in deep networks wouldn't be a good choice

Relu turns the output and its derivative to zero for negetive values this makes the noroun to have no contibution to output and also no contibution to the updating prcess so using LeakyRelu that doesn't compeletly ignores the negetive values would be a better choice.

Testing the model with different batch sizes

Choosing large or small batch size comes with pros. and cons. Large batches allow faster computaion and more reliable loss value since its mean of more instances of data but it lessen the number of time our weights are being updated. Also small batches makes the updating process more frequent but also less considrate for the whole data which may be good in case we want to exit local maxes but it's not wise to choose very small batch size.

In [21]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-2
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 16, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 16, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation=LeakyRelu(), weight_initializer='uniform')
network.add_layer(10, activation=LeakyRelu(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.6545833333333333	Average Loss: 0.9488920263795034
	Test: Average Accuracy: 0.7811	Average Loss: 0.639590394118859
Epoch 2:
	Train: Average Accuracy: 0.77305	Average Loss: 0.6558084148038149
	Test: Average Accuracy: 0.7922	Average Loss: 0.6063791197397552
Epoch 3:
	Train: Average Accuracy: 0.7844833333333333	Average Loss: 0.6313668629364496
	Test: Average Accuracy: 0.7495	Average Loss: 0.7455449469581558
Epoch 4:
	Train: Average Accuracy: 0.7895166666666666	Average Loss: 0.6132806786412112
	Test: Average Accuracy: 0.7471	Average Loss: 0.7941653065233901
Epoch 5:
	Train: Average Accuracy: 0.79575	Average Loss: 0.6060134225710195
	Test: Average Accuracy: 0.7517	Average Loss: 0.7023404007178948
Epoch 6:
	Train: Average Accuracy: 0.7978833333333334	Average Loss: 0.5974807617139655
	Test: Average Accuracy: 0.8115	Average Loss: 0.5723660452579371
Epoch 7:
	Train: Average Accuracy: 0.8006833333333333	Average Loss: 0.591977413439563
	Test: Average Accuracy: 0.8101	Average Loss: 0.5702705654585236
Epoch 8:
	Train: Average Accuracy: 0.8038	Average Loss: 0.5789036014411111
	Test: Average Accuracy: 0.7952	Average Loss: 0.6195709521090516
Epoch 9:
	Train: Average Accuracy: 0.8095333333333333	Average Loss: 0.5675116967247733
	Test: Average Accuracy: 0.7917	Average Loss: 0.6319070098567349
Epoch 10:
	Train: Average Accuracy: 0.8081166666666667	Average Loss: 0.562839041494297
	Test: Average Accuracy: 0.8135	Average Loss: 0.5612318482851523
Epoch 11:
	Train: Average Accuracy: 0.8121666666666667	Average Loss: 0.5531216760372916
	Test: Average Accuracy: 0.8146	Average Loss: 0.5550556089554546
Epoch 12:
	Train: Average Accuracy: 0.8142166666666667	Average Loss: 0.5430101186948104
	Test: Average Accuracy: 0.8143	Average Loss: 0.5559501075648957
Epoch 13:
	Train: Average Accuracy: 0.8199166666666666	Average Loss: 0.5260456421487381
	Test: Average Accuracy: 0.8079	Average Loss: 0.5665492985141185
Epoch 14:
	Train: Average Accuracy: 0.82155	Average Loss: 0.5215626225160909
	Test: Average Accuracy: 0.8065	Average Loss: 0.5492839714124073
Epoch 15:
	Train: Average Accuracy: 0.82535	Average Loss: 0.5119663595261721
	Test: Average Accuracy: 0.8146	Average Loss: 0.532999298335049
Epoch 16:
	Train: Average Accuracy: 0.8279833333333333	Average Loss: 0.4996541872545828
	Test: Average Accuracy: 0.8067	Average Loss: 0.5426834731976361
Epoch 17:
	Train: Average Accuracy: 0.8298666666666666	Average Loss: 0.4943818457389995
	Test: Average Accuracy: 0.8222	Average Loss: 0.5281702242266759
Epoch 18:
	Train: Average Accuracy: 0.8330333333333333	Average Loss: 0.48165888422628056
	Test: Average Accuracy: 0.7956	Average Loss: 0.5807275860222079
Epoch 19:
	Train: Average Accuracy: 0.8331	Average Loss: 0.475121938889332
	Test: Average Accuracy: 0.8183	Average Loss: 0.5322928111819665
Epoch 20:
	Train: Average Accuracy: 0.8356	Average Loss: 0.4704078936098523
	Test: Average Accuracy: 0.8306	Average Loss: 0.5264876628978904
Epoch 21:
	Train: Average Accuracy: 0.8396	Average Loss: 0.46367273690960664
	Test: Average Accuracy: 0.8225	Average Loss: 0.5126251729428514
Epoch 22:
	Train: Average Accuracy: 0.84025	Average Loss: 0.45934147993307706
	Test: Average Accuracy: 0.8225	Average Loss: 0.510819231009219
Epoch 23:
	Train: Average Accuracy: 0.8428333333333333	Average Loss: 0.4523804534212952
	Test: Average Accuracy: 0.8363	Average Loss: 0.48875679040783143
Epoch 24:
	Train: Average Accuracy: 0.8456166666666667	Average Loss: 0.44766693503875815
	Test: Average Accuracy: 0.8326	Average Loss: 0.5018879863483465
Epoch 25:
	Train: Average Accuracy: 0.8466333333333333	Average Loss: 0.4429182066103463
	Test: Average Accuracy: 0.8289	Average Loss: 0.51649014125962
Epoch 26:
	Train: Average Accuracy: 0.8456833333333333	Average Loss: 0.4397911912554258
	Test: Average Accuracy: 0.8225	Average Loss: 0.5394222556889406
Epoch 27:
	Train: Average Accuracy: 0.8485333333333334	Average Loss: 0.4345889879808632
	Test: Average Accuracy: 0.8373	Average Loss: 0.489975722958107
Epoch 28:
	Train: Average Accuracy: 0.8493	Average Loss: 0.43138883604479017
	Test: Average Accuracy: 0.8317	Average Loss: 0.4973882463773505
Epoch 29:
	Train: Average Accuracy: 0.8499166666666667	Average Loss: 0.42993453855502123
	Test: Average Accuracy: 0.8313	Average Loss: 0.531713158050251
Epoch 30:
	Train: Average Accuracy: 0.8502666666666666	Average Loss: 0.4284961875702997
	Test: Average Accuracy: 0.8348	Average Loss: 0.4935747992245907
In [22]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-2
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 128, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 128, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation=LeakyRelu(), weight_initializer='uniform')
network.add_layer(10, activation=LeakyRelu(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.1003772702991453	Average Loss: nan
	Test: Average Accuracy: 0.08974358974358974	Average Loss: 2.2557228398971385
Epoch 2:
	Train: Average Accuracy: 0.1007278311965812	Average Loss: 2.254665109280483
	Test: Average Accuracy: 0.0999599358974359	Average Loss: 2.252763081485271
Epoch 3:
	Train: Average Accuracy: 0.10303151709401709	Average Loss: 2.251742179055392
	Test: Average Accuracy: 0.10296474358974358	Average Loss: 2.249617526119935
Epoch 4:
	Train: Average Accuracy: 0.10730502136752136	Average Loss: 2.2484152314883254
	Test: Average Accuracy: 0.10016025641025642	Average Loss: 2.2461634127214865
Epoch 5:
	Train: Average Accuracy: 0.10939169337606838	Average Loss: 2.2446572835897993
	Test: Average Accuracy: 0.10506810897435898	Average Loss: 2.2429318822974653
Epoch 6:
	Train: Average Accuracy: 0.11485042735042734	Average Loss: 2.241227792172443
	Test: Average Accuracy: 0.12029246794871795	Average Loss: 2.2391418888358037
Epoch 7:
	Train: Average Accuracy: 0.11453325320512821	Average Loss: 2.2370033351463925
	Test: Average Accuracy: 0.11468349358974358	Average Loss: 2.2342307114095106
Epoch 8:
	Train: Average Accuracy: 0.11743790064102565	Average Loss: 2.2324815212205302
	Test: Average Accuracy: 0.10006009615384616	Average Loss: 2.230244183016148
Epoch 9:
	Train: Average Accuracy: 0.11565170940170941	Average Loss: 2.227601951076856
	Test: Average Accuracy: 0.10006009615384616	Average Loss: 2.225519207355672
Epoch 10:
	Train: Average Accuracy: 0.12316372863247864	Average Loss: 2.222356705941476
	Test: Average Accuracy: 0.10006009615384616	Average Loss: 2.220147019247912
Epoch 11:
	Train: Average Accuracy: 0.12030916132478632	Average Loss: 2.2165102887074495
	Test: Average Accuracy: 0.0999599358974359	Average Loss: 2.2141613064177603
Epoch 12:
	Train: Average Accuracy: 0.12915665064102563	Average Loss: 2.209968859883085
	Test: Average Accuracy: 0.1321113782051282	Average Loss: 2.2068875547475204
Epoch 13:
	Train: Average Accuracy: 0.13666866987179488	Average Loss: 2.202970449474065
	Test: Average Accuracy: 0.11899038461538461	Average Loss: 2.199056676674292
Epoch 14:
	Train: Average Accuracy: 0.1448150373931624	Average Loss: 2.1953517794434116
	Test: Average Accuracy: 0.12099358974358974	Average Loss: 2.19184110507796
Epoch 15:
	Train: Average Accuracy: 0.1504740918803419	Average Loss: 2.1873210345131606
	Test: Average Accuracy: 0.14413060897435898	Average Loss: 2.1839479568708455
Epoch 16:
	Train: Average Accuracy: 0.15337873931623933	Average Loss: 2.177983186656894
	Test: Average Accuracy: 0.1502403846153846	Average Loss: 2.173984997832664
Epoch 17:
	Train: Average Accuracy: 0.17115718482905984	Average Loss: 2.16821503111059
	Test: Average Accuracy: 0.17678285256410256	Average Loss: 2.165721817230201
Epoch 18:
	Train: Average Accuracy: 0.18694911858974358	Average Loss: 2.1574604194732365
	Test: Average Accuracy: 0.10056089743589744	Average Loss: 2.1536766125970233
Epoch 19:
	Train: Average Accuracy: 0.18723290598290598	Average Loss: 2.14596883169218
	Test: Average Accuracy: 0.26171875	Average Loss: 2.140801281511163
Epoch 20:
	Train: Average Accuracy: 0.20012019230769232	Average Loss: 2.1337455551703837
	Test: Average Accuracy: 0.33022836538461536	Average Loss: 2.1280285958475362
Epoch 21:
	Train: Average Accuracy: 0.2181824252136752	Average Loss: 2.120390357716246
	Test: Average Accuracy: 0.31189903846153844	Average Loss: 2.1154237716406312
Epoch 22:
	Train: Average Accuracy: 0.2359107905982906	Average Loss: 2.1065316258120292
	Test: Average Accuracy: 0.2715344551282051	Average Loss: 2.1002264906430392
Epoch 23:
	Train: Average Accuracy: 0.24759615384615385	Average Loss: 2.091405374718223
	Test: Average Accuracy: 0.2864583333333333	Average Loss: 2.0858714206494904
Epoch 24:
	Train: Average Accuracy: 0.26547475961538464	Average Loss: 2.075639305813388
	Test: Average Accuracy: 0.15925480769230768	Average Loss: 2.0709542587457186
Epoch 25:
	Train: Average Accuracy: 0.2770432692307692	Average Loss: 2.059327874121215
	Test: Average Accuracy: 0.34164663461538464	Average Loss: 2.052102156079837
Epoch 26:
	Train: Average Accuracy: 0.2924846420940171	Average Loss: 2.0420218870064266
	Test: Average Accuracy: 0.3527644230769231	Average Loss: 2.034667189643623
Epoch 27:
	Train: Average Accuracy: 0.30435363247863245	Average Loss: 2.0243630499387644
	Test: Average Accuracy: 0.3206129807692308	Average Loss: 2.0169338021491257
Epoch 28:
	Train: Average Accuracy: 0.31391893696581197	Average Loss: 2.0060666731999737
	Test: Average Accuracy: 0.27163461538461536	Average Loss: 1.9988614544654557
Epoch 29:
	Train: Average Accuracy: 0.3219150641025641	Average Loss: 1.9873073001865114
	Test: Average Accuracy: 0.2871594551282051	Average Loss: 1.9807081638496096
Epoch 30:
	Train: Average Accuracy: 0.32900974893162394	Average Loss: 1.9676251821603659
	Test: Average Accuracy: 0.3173076923076923	Average Loss: 1.9546685524166798

Overfitting

Overfitting is the process the training the model too much which makes it too specific for the train data and lose its generality when facing a new data(test data) in overfitting the result for the train data keeps growing but the result for the test data degrade at some point and this can be prevented using early stopping

In [55]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-3
EPOCHS = 200
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation=LeakyRelu(), weight_initializer='uniform')
network.add_layer(10, activation=LeakyRelu(), weight_initializer='uniform')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
est: Average Accuracy: 0.8631810897435898	Average Loss: 0.4036757549039213
Epoch 79:
	Train: Average Accuracy: 0.90625	Average Loss: 0.2614989527772561
	Test: Average Accuracy: 0.8653846153846154	Average Loss: 0.39606499580264926
Epoch 80:
	Train: Average Accuracy: 0.9046833333333333	Average Loss: 0.2602340964759822
	Test: Average Accuracy: 0.8634815705128205	Average Loss: 0.3919728981875561
Epoch 81:
	Train: Average Accuracy: 0.9064833333333333	Average Loss: 0.2602215000386633
	Test: Average Accuracy: 0.8655849358974359	Average Loss: 0.39255982172214066
Epoch 82:
	Train: Average Accuracy: 0.9067333333333333	Average Loss: 0.258914443603033
	Test: Average Accuracy: 0.862479967948718	Average Loss: 0.40244157010246223
Epoch 83:
	Train: Average Accuracy: 0.9054666666666666	Average Loss: 0.25923104479430975
	Test: Average Accuracy: 0.8657852564102564	Average Loss: 0.3883317971478957
Epoch 84:
	Train: Average Accuracy: 0.9062833333333333	Average Loss: 0.2576185366211955
	Test: Average Accuracy: 0.8625801282051282	Average Loss: 0.40381214010563765
Epoch 85:
	Train: Average Accuracy: 0.9060333333333334	Average Loss: 0.2587168395645279
	Test: Average Accuracy: 0.8616786858974359	Average Loss: 0.4075405187551229
Epoch 86:
	Train: Average Accuracy: 0.9075166666666666	Average Loss: 0.2567170721003479
	Test: Average Accuracy: 0.8609775641025641	Average Loss: 0.401329757760968
Epoch 87:
	Train: Average Accuracy: 0.90805	Average Loss: 0.2562916800369116
	Test: Average Accuracy: 0.8657852564102564	Average Loss: 0.3939703913511556
Epoch 88:
	Train: Average Accuracy: 0.9067	Average Loss: 0.25562046007482514
	Test: Average Accuracy: 0.8673878205128205	Average Loss: 0.38997348684471383
Epoch 89:
	Train: Average Accuracy: 0.9073833333333333	Average Loss: 0.25548448613673
	Test: Average Accuracy: 0.8665865384615384	Average Loss: 0.39096163200447226
Epoch 90:
	Train: Average Accuracy: 0.9072833333333333	Average Loss: 0.2542122248417969
	Test: Average Accuracy: 0.8641826923076923	Average Loss: 0.4066179641804348
Epoch 91:
	Train: Average Accuracy: 0.9088166666666667	Average Loss: 0.2531732825013256
	Test: Average Accuracy: 0.862479967948718	Average Loss: 0.4045069812141365
Epoch 92:
	Train: Average Accuracy: 0.90755	Average Loss: 0.25374240850455987
	Test: Average Accuracy: 0.8608774038461539	Average Loss: 0.41110031905070177
Epoch 93:
	Train: Average Accuracy: 0.90745	Average Loss: 0.25303823293998584
	Test: Average Accuracy: 0.8642828525641025	Average Loss: 0.4097322283768479
Epoch 94:
	Train: Average Accuracy: 0.9077666666666667	Average Loss: 0.2527146508069033
	Test: Average Accuracy: 0.8658854166666666	Average Loss: 0.39950578916550966
Epoch 95:
	Train: Average Accuracy: 0.9089833333333334	Average Loss: 0.2517591912090144
	Test: Average Accuracy: 0.8574719551282052	Average Loss: 0.412784031851468
Epoch 96:
	Train: Average Accuracy: 0.9081666666666667	Average Loss: 0.2518602968728109
	Test: Average Accuracy: 0.8701923076923077	Average Loss: 0.39359801861998533
Epoch 97:
	Train: Average Accuracy: 0.9081666666666667	Average Loss: 0.250729217832833
	Test: Average Accuracy: 0.8631810897435898	Average Loss: 0.40600273605012577
Epoch 98:
	Train: Average Accuracy: 0.9089333333333334	Average Loss: 0.2507100329063051
	Test: Average Accuracy: 0.8658854166666666	Average Loss: 0.40215209698371723
Epoch 99:
	Train: Average Accuracy: 0.9095166666666666	Average Loss: 0.24980086938492788
	Test: Average Accuracy: 0.8658854166666666	Average Loss: 0.40501669964325576
Epoch 100:
	Train: Average Accuracy: 0.9096	Average Loss: 0.24981003655642067
	Test: Average Accuracy: 0.8642828525641025	Average Loss: 0.3990429848417574
Epoch 101:
	Train: Average Accuracy: 0.9103333333333333	Average Loss: 0.24833341920682414
	Test: Average Accuracy: 0.8496594551282052	Average Loss: 0.46019914871095874
Epoch 102:
	Train: Average Accuracy: 0.9105833333333333	Average Loss: 0.24758268398958347
	Test: Average Accuracy: 0.8630809294871795	Average Loss: 0.4072015038558852
Epoch 103:
	Train: Average Accuracy: 0.9102666666666667	Average Loss: 0.2476132625082923
	Test: Average Accuracy: 0.8648838141025641	Average Loss: 0.40323063606507104
Epoch 104:
	Train: Average Accuracy: 0.9104333333333333	Average Loss: 0.24632288193166563
	Test: Average Accuracy: 0.8668870192307693	Average Loss: 0.403860568407628
Epoch 105:
	Train: Average Accuracy: 0.9097833333333334	Average Loss: 0.2468700281879139
	Test: Average Accuracy: 0.8581730769230769	Average Loss: 0.421780339564998
Epoch 106:
	Train: Average Accuracy: 0.91145	Average Loss: 0.24488552578048722
	Test: Average Accuracy: 0.8638822115384616	Average Loss: 0.4041471757343204
Epoch 107:
	Train: Average Accuracy: 0.9110833333333334	Average Loss: 0.24495409857961045
	Test: Average Accuracy: 0.8662860576923077	Average Loss: 0.40532280895577755
Epoch 108:
	Train: Average Accuracy: 0.9111833333333333	Average Loss: 0.24508638115883363
	Test: Average Accuracy: 0.8676883012820513	Average Loss: 0.40670585606642934
Epoch 109:
	Train: Average Accuracy: 0.9101666666666667	Average Loss: 0.24544678571654063
	Test: Average Accuracy: 0.8652844551282052	Average Loss: 0.4006368066792166
Epoch 110:
	Train: Average Accuracy: 0.9117333333333333	Average Loss: 0.24414929553453021
	Test: Average Accuracy: 0.8666866987179487	Average Loss: 0.4077872222573299
Epoch 111:
	Train: Average Accuracy: 0.9117	Average Loss: 0.24430113828371883
	Test: Average Accuracy: 0.8644831730769231	Average Loss: 0.4111724748333243
Epoch 112:
	Train: Average Accuracy: 0.9118166666666667	Average Loss: 0.24221278041496525
	Test: Average Accuracy: 0.8673878205128205	Average Loss: 0.4050165118108794
Epoch 113:
	Train: Average Accuracy: 0.9115	Average Loss: 0.2433248020812224
	Test: Average Accuracy: 0.8620793269230769	Average Loss: 0.4091075476847281
Epoch 114:
	Train: Average Accuracy: 0.9123166666666667	Average Loss: 0.24177036150772274
	Test: Average Accuracy: 0.8605769230769231	Average Loss: 0.4228874690389105
Epoch 115:
	Train: Average Accuracy: 0.91285	Average Loss: 0.24086778773205397
	Test: Average Accuracy: 0.8676883012820513	Average Loss: 0.40547979083126773
Epoch 116:
	Train: Average Accuracy: 0.9127833333333333	Average Loss: 0.2413431095630455
	Test: Average Accuracy: 0.8590745192307693	Average Loss: 0.41436735714878625
Epoch 117:
	Train: Average Accuracy: 0.9125666666666666	Average Loss: 0.2409117393015367
	Test: Average Accuracy: 0.8652844551282052	Average Loss: 0.41316348386388424
Epoch 118:
	Train: Average Accuracy: 0.9127833333333333	Average Loss: 0.24045475237643033
	Test: Average Accuracy: 0.8684895833333334	Average Loss: 0.4037593474404603
Epoch 119:
	Train: Average Accuracy: 0.9138	Average Loss: 0.2397966232488773
	Test: Average Accuracy: 0.8665865384615384	Average Loss: 0.41107211201949184
Epoch 120:
	Train: Average Accuracy: 0.9128833333333334	Average Loss: 0.23958637877895167
	Test: Average Accuracy: 0.8630809294871795	Average Loss: 0.4207369983564296
Epoch 121:
	Train: Average Accuracy: 0.9137	Average Loss: 0.23930070330611866
	Test: Average Accuracy: 0.8515625	Average Loss: 0.4386156459696906
Epoch 122:
	Train: Average Accuracy: 0.9144	Average Loss: 0.23924573209256747
	Test: Average Accuracy: 0.870292467948718	Average Loss: 0.40130706907421365
Epoch 123:
	Train: Average Accuracy: 0.9148333333333334	Average Loss: 0.2372520616397722
	Test: Average Accuracy: 0.8621794871794872	Average Loss: 0.4160308952432716
Epoch 124:
	Train: Average Accuracy: 0.9135	Average Loss: 0.23850887360291945
	Test: Average Accuracy: 0.8672876602564102	Average Loss: 0.4053505108156697
Epoch 125:
	Train: Average Accuracy: 0.9138166666666667	Average Loss: 0.23727296819786245
	Test: Average Accuracy: 0.8631810897435898	Average Loss: 0.4257853987745564
Epoch 126:
	Train: Average Accuracy: 0.9138333333333334	Average Loss: 0.237095074298905
	Test: Average Accuracy: 0.8652844551282052	Average Loss: 0.4129458751466285
Epoch 127:
	Train: Average Accuracy: 0.9140333333333334	Average Loss: 0.23634108099626483
	Test: Average Accuracy: 0.8659855769230769	Average Loss: 0.4128937150133949
Epoch 128:
	Train: Average Accuracy: 0.9152166666666667	Average Loss: 0.23685947255342105
	Test: Average Accuracy: 0.8676883012820513	Average Loss: 0.4120030263719941
Epoch 129:
	Train: Average Accuracy: 0.9151166666666667	Average Loss: 0.2358923038336897
	Test: Average Accuracy: 0.8676883012820513	Average Loss: 0.4128343278173385
Epoch 130:
	Train: Average Accuracy: 0.9147166666666666	Average Loss: 0.23586796203237867
	Test: Average Accuracy: 0.8655849358974359	Average Loss: 0.40481495996422306
Epoch 131:
	Train: Average Accuracy: 0.9150833333333334	Average Loss: 0.2353171132935599
	Test: Average Accuracy: 0.8589743589743589	Average Loss: 0.4337633562078719
Epoch 132:
	Train: Average Accuracy: 0.9154333333333333	Average Loss: 0.23487107251369543
	Test: Average Accuracy: 0.8628806089743589	Average Loss: 0.4266246294506662
Epoch 133:
	Train: Average Accuracy: 0.9147666666666666	Average Loss: 0.2333703841683613
	Test: Average Accuracy: 0.8590745192307693	Average Loss: 0.4220892104062566
Epoch 134:
	Train: Average Accuracy: 0.9154166666666667	Average Loss: 0.2345247460461086
	Test: Average Accuracy: 0.8678886217948718	Average Loss: 0.4130991416167484
Epoch 135:
	Train: Average Accuracy: 0.9145	Average Loss: 0.23414535877348389
	Test: Average Accuracy: 0.8629807692307693	Average Loss: 0.4245446252927904
Epoch 136:
	Train: Average Accuracy: 0.91515	Average Loss: 0.2335095977442541
	Test: Average Accuracy: 0.8673878205128205	Average Loss: 0.4182733927716485
Epoch 137:
	Train: Average Accuracy: 0.9151333333333334	Average Loss: 0.23192624452730404
	Test: Average Accuracy: 0.8657852564102564	Average Loss: 0.41556943839921745
Epoch 138:
	Train: Average Accuracy: 0.91625	Average Loss: 0.2317823511200441
	Test: Average Accuracy: 0.86328125	Average Loss: 0.42386606671260435
Epoch 139:
	Train: Average Accuracy: 0.9157166666666666	Average Loss: 0.23200265539772105
	Test: Average Accuracy: 0.8650841346153846	Average Loss: 0.4370572419044106
Epoch 140:
	Train: Average Accuracy: 0.9162333333333333	Average Loss: 0.23208790660530076
	Test: Average Accuracy: 0.8619791666666666	Average Loss: 0.43828696610293205
Epoch 141:
	Train: Average Accuracy: 0.9156833333333333	Average Loss: 0.2322220610012942
	Test: Average Accuracy: 0.8651842948717948	Average Loss: 0.42171392731191404
Epoch 142:
	Train: Average Accuracy: 0.91635	Average Loss: 0.23067915062976296
	Test: Average Accuracy: 0.8628806089743589	Average Loss: 0.4301461476989584
Epoch 143:
	Train: Average Accuracy: 0.9165833333333333	Average Loss: 0.23102534228205532
	Test: Average Accuracy: 0.8588741987179487	Average Loss: 0.43596755889026356
Epoch 144:
	Train: Average Accuracy: 0.9162833333333333	Average Loss: 0.23073748721236134
	Test: Average Accuracy: 0.8602764423076923	Average Loss: 0.42881998452193404
Epoch 145:
	Train: Average Accuracy: 0.9178	Average Loss: 0.22901592717425096
	Test: Average Accuracy: 0.86328125	Average Loss: 0.4200111458950663
Epoch 146:
	Train: Average Accuracy: 0.9163666666666667	Average Loss: 0.22957635060445575
	Test: Average Accuracy: 0.8649839743589743	Average Loss: 0.42079681789937223
Epoch 147:
	Train: Average Accuracy: 0.9165166666666666	Average Loss: 0.22923669713222794
	Test: Average Accuracy: 0.8613782051282052	Average Loss: 0.43668631717824663
Epoch 148:
	Train: Average Accuracy: 0.9168333333333333	Average Loss: 0.2298640431388873
	Test: Average Accuracy: 0.8645833333333334	Average Loss: 0.4265836256607264
Epoch 149:
	Train: Average Accuracy: 0.9166166666666666	Average Loss: 0.22918226495267452
	Test: Average Accuracy: 0.8576722756410257	Average Loss: 0.43942019191830256
Epoch 150:
	Train: Average Accuracy: 0.9169666666666667	Average Loss: 0.2294731187021306
	Test: Average Accuracy: 0.8622796474358975	Average Loss: 0.4297864065086013
Epoch 151:
	Train: Average Accuracy: 0.9178833333333334	Average Loss: 0.22848821626409585
	Test: Average Accuracy: 0.8574719551282052	Average Loss: 0.4482705854325677
Epoch 152:
	Train: Average Accuracy: 0.9173666666666667	Average Loss: 0.22792370035840134
	Test: Average Accuracy: 0.8655849358974359	Average Loss: 0.41841897940833334
Epoch 153:
	Train: Average Accuracy: 0.9172166666666667	Average Loss: 0.2283581557589394
	Test: Average Accuracy: 0.8671875	Average Loss: 0.42304522475238177
Epoch 154:
	Train: Average Accuracy: 0.9179	Average Loss: 0.22727371573800442
	Test: Average Accuracy: 0.8650841346153846	Average Loss: 0.4406460052002338
Epoch 155:
	Train: Average Accuracy: 0.91765	Average Loss: 0.2262081500704411
	Test: Average Accuracy: 0.8649839743589743	Average Loss: 0.42988992937214876
Epoch 156:
	Train: Average Accuracy: 0.9169166666666667	Average Loss: 0.2276688590324096
	Test: Average Accuracy: 0.8641826923076923	Average Loss: 0.4285340679873718
Epoch 157:
	Train: Average Accuracy: 0.9180166666666667	Average Loss: 0.226362471716421
	Test: Average Accuracy: 0.8611778846153846	Average Loss: 0.4388396798805621
Epoch 158:
	Train: Average Accuracy: 0.9187	Average Loss: 0.22592075397537065
	Test: Average Accuracy: 0.8635817307692307	Average Loss: 0.43690913519514907
Epoch 159:
	Train: Average Accuracy: 0.9184166666666667	Average Loss: 0.22571340712590093
	Test: Average Accuracy: 0.8646834935897436	Average Loss: 0.4296737163411986
Epoch 160:
	Train: Average Accuracy: 0.9182833333333333	Average Loss: 0.22504403992126612
	Test: Average Accuracy: 0.8630809294871795	Average Loss: 0.43570255815684916
Epoch 161:
	Train: Average Accuracy: 0.9179666666666667	Average Loss: 0.22511457192967457
	Test: Average Accuracy: 0.8653846153846154	Average Loss: 0.4241486873803134
Epoch 162:
	Train: Average Accuracy: 0.9181833333333334	Average Loss: 0.22534065596449684
	Test: Average Accuracy: 0.8653846153846154	Average Loss: 0.43042713526909676
Epoch 163:
	Train: Average Accuracy: 0.9174166666666667	Average Loss: 0.2250394557201779
	Test: Average Accuracy: 0.8654847756410257	Average Loss: 0.43522670438843786
Epoch 164:
	Train: Average Accuracy: 0.9189166666666667	Average Loss: 0.22372279318618832
	Test: Average Accuracy: 0.8608774038461539	Average Loss: 0.436867867353199
Epoch 165:
	Train: Average Accuracy: 0.9196166666666666	Average Loss: 0.22569819748127684
	Test: Average Accuracy: 0.8630809294871795	Average Loss: 0.4388109632247439
Epoch 166:
	Train: Average Accuracy: 0.9191166666666667	Average Loss: 0.22309744379933621
	Test: Average Accuracy: 0.8577724358974359	Average Loss: 0.46070874192227135
Epoch 167:
	Train: Average Accuracy: 0.9187166666666666	Average Loss: 0.22489954691743358
	Test: Average Accuracy: 0.8618790064102564	Average Loss: 0.43859261854985293
Epoch 168:
	Train: Average Accuracy: 0.9187833333333333	Average Loss: 0.22320749365399176
	Test: Average Accuracy: 0.8628806089743589	Average Loss: 0.4346232398647866
Epoch 169:
	Train: Average Accuracy: 0.9191833333333334	Average Loss: 0.22368092173349904
	Test: Average Accuracy: 0.856270032051282	Average Loss: 0.4421782030169767
Epoch 170:
	Train: Average Accuracy: 0.9199333333333334	Average Loss: 0.22201862212608559
	Test: Average Accuracy: 0.8667868589743589	Average Loss: 0.4354727001450215
Epoch 171:
	Train: Average Accuracy: 0.9193166666666667	Average Loss: 0.22265303756528843
	Test: Average Accuracy: 0.8617788461538461	Average Loss: 0.4365201919744061
Epoch 172:
	Train: Average Accuracy: 0.919	Average Loss: 0.22230261106292235
	Test: Average Accuracy: 0.8634815705128205	Average Loss: 0.43419631017246035
Epoch 173:
	Train: Average Accuracy: 0.9189666666666667	Average Loss: 0.22124862991899893
	Test: Average Accuracy: 0.8615785256410257	Average Loss: 0.45236914038283743
Epoch 174:
	Train: Average Accuracy: 0.9197666666666666	Average Loss: 0.22156518104380446
	Test: Average Accuracy: 0.8647836538461539	Average Loss: 0.42851127652446824
Epoch 175:
	Train: Average Accuracy: 0.91985	Average Loss: 0.22125403609337946
	Test: Average Accuracy: 0.8680889423076923	Average Loss: 0.43841334710864344
Epoch 176:
	Train: Average Accuracy: 0.9202	Average Loss: 0.22130505737188794
	Test: Average Accuracy: 0.8627804487179487	Average Loss: 0.4302590483899915
Epoch 177:
	Train: Average Accuracy: 0.9206333333333333	Average Loss: 0.22044511001472378
	Test: Average Accuracy: 0.8604767628205128	Average Loss: 0.44802288043015964
Epoch 178:
	Train: Average Accuracy: 0.9200666666666667	Average Loss: 0.22010381907535187
	Test: Average Accuracy: 0.8677884615384616	Average Loss: 0.4358633226065143
Epoch 179:
	Train: Average Accuracy: 0.9199666666666667	Average Loss: 0.21967307902024527
	Test: Average Accuracy: 0.8637820512820513	Average Loss: 0.4334514896335068
Epoch 180:
	Train: Average Accuracy: 0.9198333333333333	Average Loss: 0.2211660564122786
	Test: Average Accuracy: 0.8656850961538461	Average Loss: 0.4398677869447503
Epoch 181:
	Train: Average Accuracy: 0.9203666666666667	Average Loss: 0.2199692214598159
	Test: Average Accuracy: 0.866386217948718	Average Loss: 0.4305091545400023
Epoch 182:
	Train: Average Accuracy: 0.9211	Average Loss: 0.2194324592798141
	Test: Average Accuracy: 0.8672876602564102	Average Loss: 0.43884718893055935
Epoch 183:
	Train: Average Accuracy: 0.9203333333333333	Average Loss: 0.2195663940369571
	Test: Average Accuracy: 0.8621794871794872	Average Loss: 0.44754121054346213
Epoch 184:
	Train: Average Accuracy: 0.9210166666666667	Average Loss: 0.21885938998395005
	Test: Average Accuracy: 0.8578725961538461	Average Loss: 0.44668329382429095
Epoch 185:
	Train: Average Accuracy: 0.9205	Average Loss: 0.21867484326904435
	Test: Average Accuracy: 0.8603766025641025	Average Loss: 0.44970365420711245
Epoch 186:
	Train: Average Accuracy: 0.92145	Average Loss: 0.21779078082904377
	Test: Average Accuracy: 0.8658854166666666	Average Loss: 0.4355695383827837
Epoch 187:
	Train: Average Accuracy: 0.9198333333333333	Average Loss: 0.218819039107556
	Test: Average Accuracy: 0.8612780448717948	Average Loss: 0.4380181048306052
Epoch 188:
	Train: Average Accuracy: 0.9204	Average Loss: 0.21857789067977385
	Test: Average Accuracy: 0.8607772435897436	Average Loss: 0.4522539158776125
Epoch 189:
	Train: Average Accuracy: 0.9197	Average Loss: 0.2183968400279479
	Test: Average Accuracy: 0.8620793269230769	Average Loss: 0.45732798990251045
Epoch 190:
	Train: Average Accuracy: 0.92085	Average Loss: 0.21757550763729938
	Test: Average Accuracy: 0.8656850961538461	Average Loss: 0.44081684412724725
Epoch 191:
	Train: Average Accuracy: 0.9207166666666666	Average Loss: 0.21769779281334384
	Test: Average Accuracy: 0.8659855769230769	Average Loss: 0.4438358363125366
Epoch 192:
	Train: Average Accuracy: 0.9206666666666666	Average Loss: 0.21734392358481144
	Test: Average Accuracy: 0.8607772435897436	Average Loss: 0.4563446688469944
Epoch 193:
	Train: Average Accuracy: 0.9219333333333334	Average Loss: 0.2170495056696883
	Test: Average Accuracy: 0.858573717948718	Average Loss: 0.4513682055167644
Epoch 194:
	Train: Average Accuracy: 0.92215	Average Loss: 0.21598948643093904
	Test: Average Accuracy: 0.8605769230769231	Average Loss: 0.4452452303303235
Epoch 195:
	Train: Average Accuracy: 0.9218	Average Loss: 0.21677840540369905
	Test: Average Accuracy: 0.8623798076923077	Average Loss: 0.4460542348029637
Epoch 196:
	Train: Average Accuracy: 0.9208833333333334	Average Loss: 0.2157370158758804
	Test: Average Accuracy: 0.8641826923076923	Average Loss: 0.4358089387924009
Epoch 197:
	Train: Average Accuracy: 0.9210833333333334	Average Loss: 0.21694636659969904
	Test: Average Accuracy: 0.8622796474358975	Average Loss: 0.4507184134755957
Epoch 198:
	Train: Average Accuracy: 0.9221666666666667	Average Loss: 0.21546005914158223
	Test: Average Accuracy: 0.8627804487179487	Average Loss: 0.46408244459577785
Epoch 199:
	Train: Average Accuracy: 0.9212833333333333	Average Loss: 0.21552457235570294
	Test: Average Accuracy: 0.8625801282051282	Average Loss: 0.44341829889124357
Epoch 200:
	Train: Average Accuracy: 0.9221666666666667	Average Loss: 0.21559556148022507
	Test: Average Accuracy: 0.8670873397435898	Average Loss: 0.4456876978959586
In [68]:
plt.figure(figsize=(10, 10))
plt.plot(log['train_accuracy'], label='train')
plt.plot(log['test_accuracy'], label='test')
plt.ylim(0.8)
plt.title('Accuracy')
plt.xlabel('Epoch')
plt.ylabel('Accuracy')
plt.legend()
plt.show()
In [70]:
plt.figure(figsize=(10, 10))
plt.plot(log['train_loss'], label='train')
plt.plot(log['test_loss'], label='test')
plt.ylim(0, 1)
plt.title('Loss')
plt.xlabel('Epoch')
plt.ylabel('Loss')
plt.legend()
plt.show()

Visualization

Here we use a 2 neuron so we can plot it and see how each layer before this layer specifies a feature.

In [74]:
INPUT_SHAPE = len(ntrain_data[0])
LEARNING_RATE = 1e-2
EPOCHS = 30
TRAINLOADER = Dataloader(ntrain_data, train_labels, 10, 32, True)
TESTLOADER = Dataloader(ntest_data, test_labels, 10, 32, True)

network = FeedForwardNN(INPUT_SHAPE)
network.add_layer(20, input_shape=INPUT_SHAPE, activation=LeakyRelu(), weight_initializer='normal')
# network.add_layer(20, activation=Relu(), weight_initializer='normal')
network.add_layer(2, activation=LeakyRelu(), weight_initializer='normal')
network.add_layer(10, activation=LeakyRelu(), weight_initializer='normal')
network.set_training_param(loss=CrossEntropy(), lr=LEARNING_RATE)

log = network.fit(EPOCHS, TRAINLOADER, TESTLOADER, print_results=True)
Epoch 1:
	Train: Average Accuracy: 0.11963333333333333	Average Loss: 2.2259035806039607
	Test: Average Accuracy: 0.20012019230769232	Average Loss: 1.7538276353313267
Epoch 2:
	Train: Average Accuracy: 0.36943333333333334	Average Loss: 1.482234820602204
	Test: Average Accuracy: 0.5512820512820513	Average Loss: 1.0718933584103623
Epoch 3:
	Train: Average Accuracy: 0.5834333333333334	Average Loss: 1.0218803651547008
	Test: Average Accuracy: 0.5886418269230769	Average Loss: 1.053056394876781
Epoch 4:
	Train: Average Accuracy: 0.6167166666666667	Average Loss: 0.9593157545187724
	Test: Average Accuracy: 0.6552483974358975	Average Loss: 0.905891421433838
Epoch 5:
	Train: Average Accuracy: 0.64535	Average Loss: 0.9325154108583493
	Test: Average Accuracy: 0.6855969551282052	Average Loss: 0.8739469353778839
Epoch 6:
	Train: Average Accuracy: 0.6724	Average Loss: 0.8644021585846466
	Test: Average Accuracy: 0.6524439102564102	Average Loss: 0.8557880101764266
Epoch 7:
	Train: Average Accuracy: 0.6883833333333333	Average Loss: 0.8253680959049139
	Test: Average Accuracy: 0.6865985576923077	Average Loss: 0.8198842432286483
Epoch 8:
	Train: Average Accuracy: 0.7032166666666667	Average Loss: 0.8050436454797695
	Test: Average Accuracy: 0.7027243589743589	Average Loss: 0.7962692498475302
Epoch 9:
	Train: Average Accuracy: 0.6909	Average Loss: nan
	Test: Average Accuracy: 0.5715144230769231	Average Loss: 1.0521157978260938
Epoch 10:
	Train: Average Accuracy: 0.6292166666666666	Average Loss: nan
	Test: Average Accuracy: 0.6424278846153846	Average Loss: 0.9593629785792803
Epoch 11:
	Train: Average Accuracy: 0.6474666666666666	Average Loss: 0.9309794072834333
	Test: Average Accuracy: 0.6514423076923077	Average Loss: 0.9412691361860044
Epoch 12:
	Train: Average Accuracy: 0.6908666666666666	Average Loss: 0.8209328196027393
	Test: Average Accuracy: 0.7014222756410257	Average Loss: 0.7541563158582018
Epoch 13:
	Train: Average Accuracy: 0.7225666666666667	Average Loss: 0.7342218624878362
	Test: Average Accuracy: 0.7326722756410257	Average Loss: 0.7224957310844152
Epoch 14:
	Train: Average Accuracy: 0.7294333333333334	Average Loss: 0.7201498415484884
	Test: Average Accuracy: 0.7342748397435898	Average Loss: 0.7019885573676574
Epoch 15:
	Train: Average Accuracy: 0.7284666666666667	Average Loss: 0.7185105541373986
	Test: Average Accuracy: 0.7293669871794872	Average Loss: 0.7396163379457878
Epoch 16:
	Train: Average Accuracy: 0.7346833333333334	Average Loss: 0.7022286806710802
	Test: Average Accuracy: 0.7399839743589743	Average Loss: 0.6950825227534643
Epoch 17:
	Train: Average Accuracy: 0.73945	Average Loss: 0.6956202295601517
	Test: Average Accuracy: 0.7406850961538461	Average Loss: 0.7047364014898909
Epoch 18:
	Train: Average Accuracy: 0.74475	Average Loss: 0.6917312506704499
	Test: Average Accuracy: 0.7058293269230769	Average Loss: 0.8108060688151648
Epoch 19:
	Train: Average Accuracy: 0.7494833333333333	Average Loss: 0.6695246052726718
	Test: Average Accuracy: 0.7482972756410257	Average Loss: 0.6957503522278474
Epoch 20:
	Train: Average Accuracy: 0.7525333333333334	Average Loss: nan
	Test: Average Accuracy: 0.7336738782051282	Average Loss: nan
Epoch 21:
	Train: Average Accuracy: 0.75735	Average Loss: nan
	Test: Average Accuracy: 0.7402844551282052	Average Loss: nan
Epoch 22:
	Train: Average Accuracy: 0.7606666666666667	Average Loss: nan
	Test: Average Accuracy: 0.7467948717948718	Average Loss: nan
Epoch 23:
	Train: Average Accuracy: 0.75575	Average Loss: nan
	Test: Average Accuracy: 0.7482972756410257	Average Loss: 0.6993849648327842
Epoch 24:
	Train: Average Accuracy: 0.7606833333333334	Average Loss: 0.6494883725526377
	Test: Average Accuracy: 0.7340745192307693	Average Loss: 0.691426182108417
Epoch 25:
	Train: Average Accuracy: 0.7629333333333334	Average Loss: 0.6455762968013398
	Test: Average Accuracy: 0.74609375	Average Loss: 0.7400484132560631
Epoch 26:
	Train: Average Accuracy: 0.7670833333333333	Average Loss: nan
	Test: Average Accuracy: 0.7498998397435898	Average Loss: nan
Epoch 27:
	Train: Average Accuracy: 0.7697333333333334	Average Loss: nan
	Test: Average Accuracy: 0.7707331730769231	Average Loss: nan
Epoch 28:
	Train: Average Accuracy: 0.7626	Average Loss: nan
	Test: Average Accuracy: 0.7347756410256411	Average Loss: nan
Epoch 29:
	Train: Average Accuracy: 0.7717166666666667	Average Loss: nan
	Test: Average Accuracy: 0.7678285256410257	Average Loss: nan
Epoch 30:
	Train: Average Accuracy: 0.7635166666666666	Average Loss: nan
	Test: Average Accuracy: 0.7495993589743589	Average Loss: nan
In [75]:
train_results, test_results = network.get_outputs()
In [101]:
import seaborn as snb

colors = ['red', 'blue', 'purple', 'green', 'yellow', 'orange', 'pink', 'black', 'gray', 'cyan']
plt.figure(figsize=(10, 10))
for batch in train_results:
    x = np.array((batch[0][:, 0]))
    y = np.array((batch[0][:, 1]))
    labels = np.array( batch[1].argmax(1) ).reshape(-1)
    plt.scatter(x, y, color=[colors[labels[i]] for i in range(len(labels))])
plt.title('Train results of the 2nouron layer')
plt.show()
In [102]:
plt.figure(figsize=(10, 10))
for batch in test_results:
    x = np.array((batch[0][:, 0]))
    y = np.array((batch[0][:, 1]))
    labels = np.array( batch[1].argmax(1) ).reshape(-1)
    plt.scatter(x, y, color=[colors[labels[i]] for i in range(len(labels))])
plt.title('Test results of the 2nouron layer')
plt.show()

As we can see data with similar labels have close values their output in the 2noroun layer.